From 742d14cbd896e9ff4816fe7c8381835b79fd8402 Mon Sep 17 00:00:00 2001 From: spisarski Date: Fri, 23 Jul 2021 15:28:53 -0600 Subject: [PATCH 01/40] Initial changes for converting the TPS AE to run in K8s Created new AE image with minikube and dependencies installed Changed node setup for the lab_trial scenario to load the necessary deployments and siddhi operators for which there are still some issues --- automation/p4/tofino/variables.tf | 5 +- playbooks/env-build/siddhi/env_build.yml | 3 +- playbooks/general/setup_docker.yml | 70 +++++++++++ playbooks/{siddhi => general}/setup_jdk.yml | 0 playbooks/{siddhi => general}/setup_kafka.yml | 0 playbooks/general/setup_minikube.yml | 59 ++++++++++ playbooks/general/setup_siddhi_operator.yml | 49 ++++++++ playbooks/general/setup_source.yml | 79 +++++++------ playbooks/siddhi/docker/Dockerfile | 16 +++ .../kubernetes/kafka-trpt-ddos-detection.yaml | 68 +++++++++++ .../kubernetes/kafka-trpt-drop-clear.yaml | 65 +++++++++++ playbooks/siddhi/kubernetes/kafka.yaml | 110 ++++++++++++++++++ .../siddhi/kubernetes/trpt-to-kafka.yaml | 65 +++++++++++ .../{setup.yml => setup_siddhi_maven.yml} | 4 +- playbooks/siddhi/setup_siddhi_minikube.yml | 19 +++ .../siddhi/templates/convert_trpt.siddhi.j2 | 2 +- .../templates/minikube_service.service.j2 | 15 +++ .../simple_ddos_clear_drop.siddhi.j2 | 2 +- .../templates/simple_ddos_detection.siddhi.j2 | 2 +- playbooks/tofino/setup_minikube_siddhi_ae.yml | 72 ++++++++++++ playbooks/tofino/setup_nodes-lab_trial.yml | 28 +---- 21 files changed, 665 insertions(+), 68 deletions(-) create mode 100644 playbooks/general/setup_docker.yml rename playbooks/{siddhi => general}/setup_jdk.yml (100%) rename playbooks/{siddhi => general}/setup_kafka.yml (100%) create mode 100644 playbooks/general/setup_minikube.yml create mode 100644 playbooks/general/setup_siddhi_operator.yml create mode 100644 playbooks/siddhi/docker/Dockerfile create mode 100644 playbooks/siddhi/kubernetes/kafka-trpt-ddos-detection.yaml create mode 100644 playbooks/siddhi/kubernetes/kafka-trpt-drop-clear.yaml create mode 100644 playbooks/siddhi/kubernetes/kafka.yaml create mode 100644 playbooks/siddhi/kubernetes/trpt-to-kafka.yaml rename playbooks/siddhi/{setup.yml => setup_siddhi_maven.yml} (93%) create mode 100644 playbooks/siddhi/setup_siddhi_minikube.yml create mode 100644 playbooks/siddhi/templates/minikube_service.service.j2 create mode 100644 playbooks/tofino/setup_minikube_siddhi_ae.yml diff --git a/automation/p4/tofino/variables.tf b/automation/p4/tofino/variables.tf index 6d6472e5..25123e97 100644 --- a/automation/p4/tofino/variables.tf +++ b/automation/p4/tofino/variables.tf @@ -36,7 +36,10 @@ variable "tofino" { } } -variable "siddhi_ae_ami" {default = "ami-00876f36fe8f97733"} +// This is for a Kubernetes Siddhi environment +variable "siddhi_ae_ami" {default = "ami-0c6deef5ada5dd434"} +// This is a good Maven Siddhi environment +//variable "siddhi_ae_ami" {default = "ami-00876f36fe8f97733"} variable "switch_instance_type" {default = "t2.2xlarge"} variable "ae_instance_type" {default = "t2.2xlarge"} diff --git a/playbooks/env-build/siddhi/env_build.yml b/playbooks/env-build/siddhi/env_build.yml index a53ed116..69511097 100644 --- a/playbooks/env-build/siddhi/env_build.yml +++ b/playbooks/env-build/siddhi/env_build.yml @@ -14,4 +14,5 @@ # https://github.com/p4lang/tutorials/blob/master/vm/user-bootstrap.sh # Project and script derived in part from the script in the link above --- -- import_playbook: ../../siddhi/setup.yml +#- import_playbook: ../../siddhi/setup_siddhi_maven.yml +- import_playbook: ../../siddhi/setup_siddhi_minikube.yml diff --git a/playbooks/general/setup_docker.yml b/playbooks/general/setup_docker.yml new file mode 100644 index 00000000..c727d36c --- /dev/null +++ b/playbooks/general/setup_docker.yml @@ -0,0 +1,70 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +- hosts: "{{ hosts | default('all') }}" + gather_facts: no + become: yes + tasks: + - name: Remove apt dependecies for Docker + apt: + name: + - docker + - docker-engine + - docker.io + - containerd + - runc + state: absent + + - name: apt update + apt: + update_cache: yes + + - name: Install apt dependencies for Docker + apt: + name: + - apt-transport-https + - ca-certificates + - curl + - gnupg + - lsb-release + register: apt_rc + retries: 3 + delay: 10 + until: apt_rc is not failed + + - name: Add Docker's GPG key + shell: curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg + + - name: Add Docker apt repo + shell: | + echo \ + "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + + - name: apt update + apt: + update_cache: yes + + - name: Install apt dependencies for Docker + apt: + name: + - docker-ce + - docker-ce-cli + - containerd.io + register: apt_rc + retries: 3 + delay: 10 + until: apt_rc is not failed + + - name: Add user {{ ansible_user }} to docker group + command: "usermod -aG docker {{ ansible_user }}" diff --git a/playbooks/siddhi/setup_jdk.yml b/playbooks/general/setup_jdk.yml similarity index 100% rename from playbooks/siddhi/setup_jdk.yml rename to playbooks/general/setup_jdk.yml diff --git a/playbooks/siddhi/setup_kafka.yml b/playbooks/general/setup_kafka.yml similarity index 100% rename from playbooks/siddhi/setup_kafka.yml rename to playbooks/general/setup_kafka.yml diff --git a/playbooks/general/setup_minikube.yml b/playbooks/general/setup_minikube.yml new file mode 100644 index 00000000..025382f0 --- /dev/null +++ b/playbooks/general/setup_minikube.yml @@ -0,0 +1,59 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- import_playbook: setup_docker.yml + +- hosts: "{{ hosts | default('all') }}" + gather_facts: no + tasks: + - name: Create minikube binary directory + become: yes + file: + path: /etc/minikube + state: directory + + - name: Get minikube + become: yes + get_url: + url: https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64 + dest: /etc/minikube/minikube-linux-amd64 + + - name: Install minikube + become: yes + command: install minikube-linux-amd64 /usr/bin/minikube + args: + chdir: /etc/minikube + + - name: Install kubectl + become: yes + command: snap install kubectl --classic + + - name: Start minikube to ensure it will work + shell: sudo -u $USER /usr/bin/minikube start + register: minkube_start_rc + async: 180 + retries: 2 + delay: 5 + until: minkube_start_rc is not failed + + - name: Check Minikube K8s cluster is operational + command: kubectl get po -A + register: kube_pods_out + + - name: Show pods + debug: + var: kube_pods_out + + - name: Stop minikube + shell: sudo -u $USER /usr/bin/minikube stop diff --git a/playbooks/general/setup_siddhi_operator.yml b/playbooks/general/setup_siddhi_operator.yml new file mode 100644 index 00000000..9866ffe9 --- /dev/null +++ b/playbooks/general/setup_siddhi_operator.yml @@ -0,0 +1,49 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- import_playbook: setup_minikube.yml + +- hosts: "{{ hosts | default('all') }}" + gather_facts: no + tasks: + - name: Start minikube to install siddhi-operator + shell: sudo -u $USER /usr/bin/minikube start + register: minkube_start_rc + async: 180 + retries: 2 + delay: 5 + until: minkube_start_rc is not failed + + - name: Install siddhi K8s operator + command: "kubectl apply -f {{ item }}" + loop: + - https://raw.githubusercontent.com/siddhi-io/siddhi-operator/master/deploy/siddhi_v1alpha2_siddhiprocess_crd.yaml + - https://raw.githubusercontent.com/siddhi-io/siddhi-operator/master/deploy/service_account.yaml + - https://raw.githubusercontent.com/siddhi-io/siddhi-operator/master/deploy/role.yaml + - https://raw.githubusercontent.com/siddhi-io/siddhi-operator/master/deploy/role_binding.yaml + - https://raw.githubusercontent.com/siddhi-io/siddhi-operator/master/deploy/operator.yaml + + - name: Check siddhi-operator is Running + command: kubectl get po + register: kube_pods_out + retries: 10 + delay: 5 + until: kube_pods_out.stdout.find("Running") != -1 + + - name: Show pods + debug: + var: kube_pods_out.stdout_lines + + - name: Stop minikube + shell: sudo -u $USER /usr/bin/minikube stop diff --git a/playbooks/general/setup_source.yml b/playbooks/general/setup_source.yml index ae21c9e7..5b5baef4 100644 --- a/playbooks/general/setup_source.yml +++ b/playbooks/general/setup_source.yml @@ -20,48 +20,51 @@ dest_dir: "/home/{{ ansible_user }}/transparent-security" run_tests: "{{ python_unit_tests | default(false) }}" install_python: "{{ install_tps_python | default(true) }}" + install_tps_dependencies: "{{ install_dependencies | default(true) }}" tasks: - - name: Install apt dependencies - apt: - update_cache: yes - name: - - python3-pip - - arping - - iperf3 - register: apt_rc - retries: 3 - delay: 10 - until: apt_rc is not failed - when: ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu' + - block: + - name: Install apt dependencies + apt: + update_cache: yes + name: + - python3-pip + - arping + - iperf3 + register: apt_rc + retries: 3 + delay: 10 + until: apt_rc is not failed + when: ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu' - - name: Install yum dependencies - yum: - update_cache: yes - name: - - python3-pip - - python3-devel - - gcc-c++ - - tcpdump - - net-tools - - iputils - - iperf3 - register: apt_rc - retries: 3 - delay: 10 - until: apt_rc is not failed - when: ansible_distribution == 'CentOS' or ansible_distribution == 'Red Hat Enterprise Linux' + - name: Install yum dependencies + yum: + update_cache: yes + name: + - python3-pip + - python3-devel + - gcc-c++ + - tcpdump + - net-tools + - iputils + - iperf3 + register: apt_rc + retries: 3 + delay: 10 + until: apt_rc is not failed + when: ansible_distribution == 'CentOS' or ansible_distribution == 'Red Hat Enterprise Linux' - - name: Downgrade pip3 scapy version to 2.4.3 due to 2.4.4 bug when running receive_packets.py on centos - pip: - name: - - scapy==2.4.3 - executable: pip3 - register: pip_rc - retries: 3 - delay: 10 - until: pip_rc is not failed - when: ansible_distribution == 'CentOS' or ansible_distribution == 'Red Hat Enterprise Linux' + - name: Downgrade pip3 scapy version to 2.4.3 due to 2.4.4 bug when running receive_packets.py on centos + pip: + name: + - scapy==2.4.3 + executable: pip3 + register: pip_rc + retries: 3 + delay: 10 + until: pip_rc is not failed + when: ansible_distribution == 'CentOS' or ansible_distribution == 'Red Hat Enterprise Linux' + when: install_tps_dependencies | bool - name: Copy local transparent-security source become: no diff --git a/playbooks/siddhi/docker/Dockerfile b/playbooks/siddhi/docker/Dockerfile new file mode 100644 index 00000000..0cf08ce4 --- /dev/null +++ b/playbooks/siddhi/docker/Dockerfile @@ -0,0 +1,16 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM siddhiio/siddhi-runner-alpine:5.1.2 +ARG classpath_lib=. +COPY $classpath_lib/lib/* /home/siddhi_user/siddhi-runner/lib/ diff --git a/playbooks/siddhi/kubernetes/kafka-trpt-ddos-detection.yaml b/playbooks/siddhi/kubernetes/kafka-trpt-ddos-detection.yaml new file mode 100644 index 00000000..77f19f68 --- /dev/null +++ b/playbooks/siddhi/kubernetes/kafka-trpt-ddos-detection.yaml @@ -0,0 +1,68 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: siddhi.io/v1alpha2 +kind: SiddhiProcess +metadata: + name: kafka-trpt-ddos-detect +spec: + apps: + - script: | + @App:name("KafkaSourcePacketJSON") + @source( + type="kafka", + topic.list="trptPacket", + bootstrap.servers="kafka-service:9092", + group.id="test", + threading.option="single.thread", + @map( + type="p4-trpt", + @attributes( + src_mac="intHdr.mdStackHdr.origMac", + ip_ver="ipHdr.version", + dst_ip="ipHdr.dstAddr", + dst_port="protoHdr.dstPort" + ) + ) + ) + define stream trptPktStream ( + src_mac string, + ip_ver int, + dst_ip string, + dst_port long); + + @sink( + type="http", + publisher.url="http://localhost:9998/aggAttack", + method="POST", + headers="trp:headers", + @map(type="json") + ) + define stream attackStream ( + src_mac string, + ip_ver int, + dst_ip string, + dst_port long, + count long + ); + + @info(name = "trptJsonQuery") + from trptPktStream#window.time(1 sec) + select src_mac, ip_ver, dst_ip, dst_port, count(ip_ver) as count + group by src_mac, dst_ip, dst_port + having count == 10 + insert into attackStream; + + container: + image: "spisarski/siddhi" + imagePullPolicy: Always diff --git a/playbooks/siddhi/kubernetes/kafka-trpt-drop-clear.yaml b/playbooks/siddhi/kubernetes/kafka-trpt-drop-clear.yaml new file mode 100644 index 00000000..0be12801 --- /dev/null +++ b/playbooks/siddhi/kubernetes/kafka-trpt-drop-clear.yaml @@ -0,0 +1,65 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: siddhi.io/v1alpha2 +kind: SiddhiProcess +metadata: + name: kafka-trpt-drop-clear +spec: + apps: + - script: | + @App:name("KafkaSourceDropJSON") + @source( + type="kafka", + topic.list="trptDrop", + bootstrap.servers="kafka-service:9092", + group.id="test", + threading.option="single.thread", + @map( + type="p4-trpt", + @attributes( + timestamp="dropHdr.timestamp", + dropKey="dropHdr.dropKey", + dropCount="dropHdr.dropCount" + ) + ) + ) + define stream trptDropStream ( + timestamp long, + dropKey string, + dropCount long + ); + + @sink( + type="http", + publisher.url="http://localhost:9998/aggAttack", + method="DELETE", + headers="trp:headers", + @map(type="json") + ) + define stream dropAttackStream ( + dropKey string, + dropCount long, + count long + ); + + @info(name = "trptJsonQuery") + from trptDropStream#window.time(35 sec) + select dropKey, dropCount, count(dropCount) as count + group by dropKey, dropCount + having count >= 3 + insert into dropAttackStream; + + container: + image: "spisarski/siddhi" + imagePullPolicy: Always diff --git a/playbooks/siddhi/kubernetes/kafka.yaml b/playbooks/siddhi/kubernetes/kafka.yaml new file mode 100644 index 00000000..ad6b6629 --- /dev/null +++ b/playbooks/siddhi/kubernetes/kafka.yaml @@ -0,0 +1,110 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: zookeeper-deploy +spec: + replicas: 2 + selector: + matchLabels: + app: zookeeper-1 + template: + metadata: + labels: + app: zookeeper-1 + spec: + containers: + - name: zoo1 + image: digitalwonderland/zookeeper + ports: + - containerPort: 2181 + env: + - name: ZOOKEEPER_ID + value: "1" + - name: ZOOKEEPER_SERVER_1 + value: zoo1 + +--- +apiVersion: v1 +kind: Service +metadata: + name: zoo1 + labels: + app: zookeeper-1 +spec: + ports: + - name: client + port: 2181 + protocol: TCP + - name: follower + port: 2888 + protocol: TCP + - name: leader + port: 3888 + protocol: TCP + selector: + app: zookeeper-1 + +--- +apiVersion: v1 +kind: Service +metadata: + name: kafka-service + labels: + name: kafka +spec: + ports: + - port: 9092 + name: kafka-port + protocol: TCP + selector: + app: kafka + id: "0" + type: LoadBalancer + +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: kafka-broker0 +spec: + replicas: 2 + selector: + matchLabels: + app: kafka + id: "0" + template: + metadata: + labels: + app: kafka + id: "0" + spec: + containers: + - name: kafka + image: wurstmeister/kafka + ports: + - containerPort: 9092 + env: + - name: KAFKA_ADVERTISED_PORT + value: "9092" + - name: KAFKA_ADVERTISED_HOST_NAME + value: kafka + - name: KAFKA_ZOOKEEPER_CONNECT + value: zoo1:2181 + - name: KAFKA_BROKER_ID + value: "0" + - name: KAFKA_CREATE_TOPICS + value: admintome-test:1:1 diff --git a/playbooks/siddhi/kubernetes/trpt-to-kafka.yaml b/playbooks/siddhi/kubernetes/trpt-to-kafka.yaml new file mode 100644 index 00000000..1c240575 --- /dev/null +++ b/playbooks/siddhi/kubernetes/trpt-to-kafka.yaml @@ -0,0 +1,65 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: siddhi.io/v1alpha2 +kind: SiddhiProcess +metadata: + name: trpt-to-kafka +spec: + apps: + - script: | + @App:name("UDPSourceTRPT") + @source( + type="udp", + listen.port="556", + @map( + type="p4-trpt", + @attributes( + in_type="telemRptHdr.inType", + full_json="jsonString" + ) + ) + ) + define stream trptUdpStream (in_type int, full_json object); + + @sink( + type="kafka", + topic="trptPacket", + bootstrap.servers="kafka-service:9092", + is.binary.message = "false", + @map(type="text") + ) + define stream trptPacket (full_json object); + + @sink( + type="kafka", + topic="trptDrop", + bootstrap.servers="kafka-service:9092", + is.binary.message = "false", + @map(type="text") + ) + define stream trptDrop (full_json object); + + @info(name = "TrptPacket") + from trptUdpStream[in_type != 2] + select full_json + insert into trptPacket; + + @info(name = "TrptDrop") + from trptUdpStream[in_type == 2] + select full_json + insert into trptDrop; + + container: + image: "spisarski/siddhi" + imagePullPolicy: Always diff --git a/playbooks/siddhi/setup.yml b/playbooks/siddhi/setup_siddhi_maven.yml similarity index 93% rename from playbooks/siddhi/setup.yml rename to playbooks/siddhi/setup_siddhi_maven.yml index 8be739bc..0871d18d 100644 --- a/playbooks/siddhi/setup.yml +++ b/playbooks/siddhi/setup_siddhi_maven.yml @@ -18,8 +18,8 @@ # ctrl-C will gracefully exit. # mvn exec:java -Dexec.mainClass=io.siddhi.extension.map.p4.StartSiddhiRuntime "-Dexec.args=/home/ubuntu/siddhi-map-p4-trpt/docs/siddhi/examples/convert_trpt.siddhi /home/ubuntu/siddhi-map-p4-trpt/docs/siddhi/examples/simple_ddos_detection.siddhi" -f pom.xml --- -- import_playbook: setup_jdk.yml -- import_playbook: setup_kafka.yml +- import_playbook: ../general/setup_jdk.yml +- import_playbook: ../general/setup_kafka.yml - import_playbook: setup_siddhi_p4.yml - import_playbook: ../general/setup_source.yml - import_playbook: ../general/final_env_setup.yml diff --git a/playbooks/siddhi/setup_siddhi_minikube.yml b/playbooks/siddhi/setup_siddhi_minikube.yml new file mode 100644 index 00000000..3e0926d0 --- /dev/null +++ b/playbooks/siddhi/setup_siddhi_minikube.yml @@ -0,0 +1,19 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Playbook that is responsible for putting together an AWS image for running +# Siddhi with the new "udp" source & P4 Telemetry report mapping extensions +# running in Kubernetes +--- +- import_playbook: ../general/setup_siddhi_operator.yml +- import_playbook: ../general/final_env_setup.yml diff --git a/playbooks/siddhi/templates/convert_trpt.siddhi.j2 b/playbooks/siddhi/templates/convert_trpt.siddhi.j2 index 468b7083..f2a33720 100644 --- a/playbooks/siddhi/templates/convert_trpt.siddhi.j2 +++ b/playbooks/siddhi/templates/convert_trpt.siddhi.j2 @@ -13,7 +13,7 @@ * limitations under the License. */ -@app:name('UDP-Source-TRPT') +@App:name('UDPSourceTRPT') @source(type='udp', listen.port='{{ telem_rpt_port }}', @map(type='p4-trpt', @attributes(in_type='telemRptHdr.inType', full_json='jsonString'))) define stream trptUdpStream (in_type int, full_json object); diff --git a/playbooks/siddhi/templates/minikube_service.service.j2 b/playbooks/siddhi/templates/minikube_service.service.j2 new file mode 100644 index 00000000..17b89ea9 --- /dev/null +++ b/playbooks/siddhi/templates/minikube_service.service.j2 @@ -0,0 +1,15 @@ +[Unit] +Description=Minikube service for TPS AE +Requires=network.target +After=syslog.target network.target + +[Service] +Type=oneshot +RemainAfterExit=yes +User=ubuntu +Group=docker +ExecStart=/usr/bin/minikube start +ExecStop=/usr/bin/minikube stop + +[Install] +WantedBy=multi-user.target diff --git a/playbooks/siddhi/templates/simple_ddos_clear_drop.siddhi.j2 b/playbooks/siddhi/templates/simple_ddos_clear_drop.siddhi.j2 index c4117754..eb1a7c6c 100644 --- a/playbooks/siddhi/templates/simple_ddos_clear_drop.siddhi.j2 +++ b/playbooks/siddhi/templates/simple_ddos_clear_drop.siddhi.j2 @@ -13,7 +13,7 @@ * limitations under the License. */ -@app:name('Kafka-Source-Drop-JSON') +@App:name('KafkaSourceDropJSON') @source(type='kafka', topic.list='{{ kafka_trpt_drop_topic }}', bootstrap.servers='{{ kafka_host_port }}', group.id='test', threading.option='single.thread', diff --git a/playbooks/siddhi/templates/simple_ddos_detection.siddhi.j2 b/playbooks/siddhi/templates/simple_ddos_detection.siddhi.j2 index 54b97080..072cbe12 100644 --- a/playbooks/siddhi/templates/simple_ddos_detection.siddhi.j2 +++ b/playbooks/siddhi/templates/simple_ddos_detection.siddhi.j2 @@ -13,7 +13,7 @@ * limitations under the License. */ -@app:name('Kafka-Source-Packet-JSON') +@App:name('KafkaSourcePacketJSON') @source(type='kafka', topic.list='{{ kafka_trpt_pkt_topic }}', bootstrap.servers='{{ kafka_host_port }}', group.id='test', threading.option='single.thread', diff --git a/playbooks/tofino/setup_minikube_siddhi_ae.yml b/playbooks/tofino/setup_minikube_siddhi_ae.yml new file mode 100644 index 00000000..f3b35832 --- /dev/null +++ b/playbooks/tofino/setup_minikube_siddhi_ae.yml @@ -0,0 +1,72 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- hosts: "{{ host_val | default('all') }}" + gather_facts: no + tasks: + - name: Create Service File to /etc/systemd/system/tps-tofino-ae.service + become: yes + template: + src: ../siddhi/templates/minikube_service.service.j2 + dest: /etc/systemd/system/tps-tofino-ae.service + + - name: Start tps-tofino-ae.service + become: yes + systemd: + name: tps-tofino-ae.service + state: restarted + enabled: yes + register: minkube_start_rc + retries: 3 + delay: 5 + until: minkube_start_rc is not failed + + - name: Wait for tps-tofino-ae service to be completely up... + command: kubectl get po + register: kube_pods_out + retries: 10 + delay: 5 + until: kube_pods_out.stdout.find("Running") != -1 + + - name: Show pods + debug: + var: kube_pods_out.stdout_lines + + - name: Install siddhi K8s operator + command: "kubectl apply -f {{ item }}" + loop: + - "{{ trans_sec_dir }}/playbooks/siddhi/kubernetes/kafka.yaml" + - "{{ trans_sec_dir }}/playbooks/siddhi/kubernetes/trpt-to-kafka.yaml" + - "{{ trans_sec_dir }}/playbooks/siddhi/kubernetes/kafka-trpt-ddos-detection.yaml" + - "{{ trans_sec_dir }}/playbooks/siddhi/kubernetes/kafka-trpt-drop-clear.yaml" + + # TODO - Improve validation by for all containing "Running" + - name: Wait for K8s pods to come up... + command: kubectl get po + register: kube_pods_out + retries: 10 + delay: 10 + until: kube_pods_out.stdout.find("Running") != -1 and + kube_pods_out.stdout.find("Creating") == -1 and + kube_pods_out.stdout_lines | length >= 9 + + - name: Show pods + debug: + var: kube_pods_out.stdout_lines + + - name: Stop tps-tofino-ae.service + become: yes + systemd: + name: tps-tofino-ae.service + state: stopped diff --git a/playbooks/tofino/setup_nodes-lab_trial.yml b/playbooks/tofino/setup_nodes-lab_trial.yml index 1216ef11..135c9750 100644 --- a/playbooks/tofino/setup_nodes-lab_trial.yml +++ b/playbooks/tofino/setup_nodes-lab_trial.yml @@ -19,6 +19,11 @@ trans_sec_source_dir: "{{ orch_trans_sec_dir }}" python_unit_tests: false +# Setup AE +- import_playbook: setup_minikube_siddhi_ae.yml + vars: + host_val: ae + # Create virtual interfaces on switches - import_playbook: setup_virt_eth.yml vars: @@ -45,26 +50,3 @@ port_to_wait: "{{ sdn_port }}" wait_timeout: 60 load_p4: False - -# Start AE -- import_playbook: ../general/start_service.yml - vars: - host_val: ae - service_name: tps-tofino-ae - srvc_start_pause_time: 45 - local_srvc_script_tmplt_file: "{{ orch_trans_sec_dir }}/playbooks/general/templates/siddhi_p4_service.sh.j2" - srvc_type: SIMPLE - sdn_url: "http://{{ sdn_ip }}:{{ sdn_port }}" - telem_rpt_port: 556 - kafka_host_port: localhost:9092 - kafka_trpt_pkt_topic: trptPacket - kafka_trpt_drop_topic: trptDrop - alert_pkt_count: 10 - alert_window_secs: 1 - templates: - - src: "{{ orch_trans_sec_dir }}/playbooks/siddhi/templates/convert_trpt.siddhi.j2" - dest: "{{ remote_scripts_dir }}/convert_trpt.siddhi" - - src: "{{ orch_trans_sec_dir }}/playbooks/siddhi/templates/simple_ddos_detection.siddhi.j2" - dest: "{{ remote_scripts_dir }}/simple_ddos_detection.siddhi" - - src: "{{ orch_trans_sec_dir }}/playbooks/siddhi/templates/simple_ddos_clear_drop.siddhi.j2" - dest: "{{ remote_scripts_dir }}/simple_ddos_clear.siddhi" From 218cb69e3ade885b9e2819b0e171f61639a2528e Mon Sep 17 00:00:00 2001 From: spisarski Date: Wed, 28 Jul 2021 15:05:22 -0600 Subject: [PATCH 02/40] added utilities and sudo password for debugging to custom siddhi runner image --- playbooks/siddhi/docker/Dockerfile | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/playbooks/siddhi/docker/Dockerfile b/playbooks/siddhi/docker/Dockerfile index 0cf08ce4..fd7e412f 100644 --- a/playbooks/siddhi/docker/Dockerfile +++ b/playbooks/siddhi/docker/Dockerfile @@ -12,5 +12,16 @@ # limitations under the License. FROM siddhiio/siddhi-runner-alpine:5.1.2 + +RUN apk --update add net-tools tcpdump sudo +RUN apk update && apk add ca-certificates && rm -rf /var/cache/apk/* +#RUN set -ex && apk --no-cache add sudo + +#SHELL ["/bin/bash", "-o", "pipefail", "-c"] +#RUN addgroup siddhi_user sudo +RUN echo 'siddhi_user:siddhi_user' | chpasswd +RUN echo '%wheel ALL=(ALL) ALL' > /etc/sudoers.d/wheel +RUN adduser siddhi_user wheel + ARG classpath_lib=. COPY $classpath_lib/lib/* /home/siddhi_user/siddhi-runner/lib/ From bfb675c7f87758094819cc6b9b3b8a89fd88fcd3 Mon Sep 17 00:00:00 2001 From: spisarski Date: Wed, 28 Jul 2021 15:11:17 -0600 Subject: [PATCH 03/40] housekeeping --- playbooks/tofino/setup_nodes-lab_trial.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/playbooks/tofino/setup_nodes-lab_trial.yml b/playbooks/tofino/setup_nodes-lab_trial.yml index 135c9750..7f343883 100644 --- a/playbooks/tofino/setup_nodes-lab_trial.yml +++ b/playbooks/tofino/setup_nodes-lab_trial.yml @@ -11,6 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +--- # Copy TPS source from orchestrator to all hosts - import_playbook: ../general/setup_source.yml vars: From 532e9903aac64ec948f733361a7dd29deb477949 Mon Sep 17 00:00:00 2001 From: spisarski Date: Wed, 28 Jul 2021 15:35:20 -0600 Subject: [PATCH 04/40] changed siddhi-operator version from master to 0.2.2 added ingress addon added siddhi ingress IP to /etc/hosts --- playbooks/general/setup_siddhi_operator.yml | 30 +++++++++++++++++---- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/playbooks/general/setup_siddhi_operator.yml b/playbooks/general/setup_siddhi_operator.yml index 9866ffe9..a4ff6fc6 100644 --- a/playbooks/general/setup_siddhi_operator.yml +++ b/playbooks/general/setup_siddhi_operator.yml @@ -28,11 +28,31 @@ - name: Install siddhi K8s operator command: "kubectl apply -f {{ item }}" loop: - - https://raw.githubusercontent.com/siddhi-io/siddhi-operator/master/deploy/siddhi_v1alpha2_siddhiprocess_crd.yaml - - https://raw.githubusercontent.com/siddhi-io/siddhi-operator/master/deploy/service_account.yaml - - https://raw.githubusercontent.com/siddhi-io/siddhi-operator/master/deploy/role.yaml - - https://raw.githubusercontent.com/siddhi-io/siddhi-operator/master/deploy/role_binding.yaml - - https://raw.githubusercontent.com/siddhi-io/siddhi-operator/master/deploy/operator.yaml + - https://github.com/siddhi-io/siddhi-operator/releases/download/v0.2.2/00-prereqs.yaml + - https://github.com/siddhi-io/siddhi-operator/releases/download/v0.2.2/01-siddhi-operator.yaml + + - name: Enable minikube ingress addon + command: sudo -u $USER /usr/bin/minikube addons enable ingress + register: ing_out + async: 60 + retries: 2 + delay: 5 + until: ing_out is not failed + + - name: Get minikube ip + command: sudo -u $USER /usr/bin/minikube ip + register: ip_out + retries: 2 + delay: 5 + until: ip_out is not failed + + - name: minikube IP + debug: + var: ip_out + + - name: Setup minikube/siddhi IP into /etc/hosts + become: yes + shell: "echo ' {{ ip_out.stdout_lines[0] }} siddhi' >> /etc/hosts" - name: Check siddhi-operator is Running command: kubectl get po From 40118d3dec6399923d1ff826642219d574561031 Mon Sep 17 00:00:00 2001 From: spisarski Date: Wed, 28 Jul 2021 15:50:36 -0600 Subject: [PATCH 05/40] updated siddhi AE image --- automation/p4/tofino/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automation/p4/tofino/variables.tf b/automation/p4/tofino/variables.tf index 25123e97..367b32b6 100644 --- a/automation/p4/tofino/variables.tf +++ b/automation/p4/tofino/variables.tf @@ -37,7 +37,7 @@ variable "tofino" { } // This is for a Kubernetes Siddhi environment -variable "siddhi_ae_ami" {default = "ami-0c6deef5ada5dd434"} +variable "siddhi_ae_ami" {default = "ami-0b339255a9e7a680e"} // This is a good Maven Siddhi environment //variable "siddhi_ae_ami" {default = "ami-00876f36fe8f97733"} From 8cee323fa8439378a43e6e3eb8e3901e7547d6de Mon Sep 17 00:00:00 2001 From: spisarski Date: Fri, 30 Jul 2021 15:17:44 -0600 Subject: [PATCH 06/40] Example K8s CRDs --- .../kafka-trpt-ddos-detection.yaml | 21 +++++- .../kafka-trpt-drop-clear.yaml | 19 +++++ .../siddhi/kubernetes => examples}/kafka.yaml | 71 +++++++++++-------- examples/test-http.yaml | 36 ++++++++++ examples/test-networking.yaml | 36 ++++++++++ examples/trpt-pv.yaml | 15 ++++ .../trpt-to-kafka.yaml | 25 ++++++- 7 files changed, 191 insertions(+), 32 deletions(-) rename {playbooks/siddhi/kubernetes => examples}/kafka-trpt-ddos-detection.yaml (80%) rename {playbooks/siddhi/kubernetes => examples}/kafka-trpt-drop-clear.yaml (82%) rename {playbooks/siddhi/kubernetes => examples}/kafka.yaml (52%) create mode 100644 examples/test-http.yaml create mode 100644 examples/test-networking.yaml create mode 100644 examples/trpt-pv.yaml rename {playbooks/siddhi/kubernetes => examples}/trpt-to-kafka.yaml (78%) diff --git a/playbooks/siddhi/kubernetes/kafka-trpt-ddos-detection.yaml b/examples/kafka-trpt-ddos-detection.yaml similarity index 80% rename from playbooks/siddhi/kubernetes/kafka-trpt-ddos-detection.yaml rename to examples/kafka-trpt-ddos-detection.yaml index 77f19f68..43cbf4bf 100644 --- a/playbooks/siddhi/kubernetes/kafka-trpt-ddos-detection.yaml +++ b/examples/kafka-trpt-ddos-detection.yaml @@ -11,6 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +--- apiVersion: siddhi.io/v1alpha2 kind: SiddhiProcess metadata: @@ -22,7 +23,7 @@ spec: @source( type="kafka", topic.list="trptPacket", - bootstrap.servers="kafka-service:9092", + bootstrap.servers="10.110.95.121:9092", group.id="test", threading.option="single.thread", @map( @@ -66,3 +67,21 @@ spec: container: image: "spisarski/siddhi" imagePullPolicy: Always + + persistentVolumeClaim: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: standard + volumeMode: Filesystem + + runner: | + state.persistence: + enabled: true + intervalInMin: 5 + revisionsToKeep: 2 + persistenceStore: io.siddhi.distribution.core.persistence.FileSystemPersistenceStore + config: + location: siddhi-app-persistence diff --git a/playbooks/siddhi/kubernetes/kafka-trpt-drop-clear.yaml b/examples/kafka-trpt-drop-clear.yaml similarity index 82% rename from playbooks/siddhi/kubernetes/kafka-trpt-drop-clear.yaml rename to examples/kafka-trpt-drop-clear.yaml index 0be12801..47788873 100644 --- a/playbooks/siddhi/kubernetes/kafka-trpt-drop-clear.yaml +++ b/examples/kafka-trpt-drop-clear.yaml @@ -11,6 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +--- apiVersion: siddhi.io/v1alpha2 kind: SiddhiProcess metadata: @@ -63,3 +64,21 @@ spec: container: image: "spisarski/siddhi" imagePullPolicy: Always + + persistentVolumeClaim: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: standard + volumeMode: Filesystem + + runner: | + state.persistence: + enabled: true + intervalInMin: 5 + revisionsToKeep: 2 + persistenceStore: io.siddhi.distribution.core.persistence.FileSystemPersistenceStore + config: + location: siddhi-app-persistence diff --git a/playbooks/siddhi/kubernetes/kafka.yaml b/examples/kafka.yaml similarity index 52% rename from playbooks/siddhi/kubernetes/kafka.yaml rename to examples/kafka.yaml index ad6b6629..59c6a169 100644 --- a/playbooks/siddhi/kubernetes/kafka.yaml +++ b/examples/kafka.yaml @@ -28,14 +28,17 @@ spec: spec: containers: - name: zoo1 +# image: bitnami/zookeeper image: digitalwonderland/zookeeper +# image: zookeeper +# imagePullPolicy: Always ports: - - containerPort: 2181 + - containerPort: 2181 env: - - name: ZOOKEEPER_ID - value: "1" - - name: ZOOKEEPER_SERVER_1 - value: zoo1 + - name: ZOOKEEPER_ID + value: "1" + - name: ZOOKEEPER_SERVER_1 + value: zoo1 --- apiVersion: v1 @@ -46,15 +49,15 @@ metadata: app: zookeeper-1 spec: ports: - - name: client - port: 2181 - protocol: TCP - - name: follower - port: 2888 - protocol: TCP - - name: leader - port: 3888 - protocol: TCP + - name: client + port: 2181 + protocol: TCP + - name: follower + port: 2888 + protocol: TCP + - name: leader + port: 3888 + protocol: TCP selector: app: zookeeper-1 @@ -67,13 +70,14 @@ metadata: name: kafka spec: ports: - - port: 9092 - name: kafka-port - protocol: TCP + - port: 9092 + targetPort: 9092 + protocol: TCP selector: app: kafka - id: "0" +# id: "0" type: LoadBalancer +# clusterIP: 10.96.1.2 --- kind: Deployment @@ -95,16 +99,25 @@ spec: containers: - name: kafka image: wurstmeister/kafka +# image: bitnami/kafka ports: - - containerPort: 9092 + - containerPort: 9092 env: - - name: KAFKA_ADVERTISED_PORT - value: "9092" - - name: KAFKA_ADVERTISED_HOST_NAME - value: kafka - - name: KAFKA_ZOOKEEPER_CONNECT - value: zoo1:2181 - - name: KAFKA_BROKER_ID - value: "0" - - name: KAFKA_CREATE_TOPICS - value: admintome-test:1:1 + - name: KAFKA_ADVERTISED_PORT + value: "30718" + - name: KAFKA_ADVERTISED_HOST_NAME + value: 10.96.1.2 + - name: KAFKA_ZOOKEEPER_CONNECT + value: zoo1:2181 + - name: KAFKA_BROKER_ID + value: "0" +# - name: KAFKA_CREATE_TOPICS +# value: admintome-test:1:1 +# - name: KAFKA_LISTENERS +# value: LISTENER_PUBLIC://kafka-service:29092,LISTENER_INTERNAL://localhost:9092 +# - name: KAFKA_ADVERTISED_LISTENERS +# value: LISTENER_PUBLIC://kafka-service:29092,LISTENER_INTERNAL://localhost:9092 +# - name: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP +# value: LISTENER_PUBLIC:PLAINTEXT,LISTENER_INTERNAL:PLAINTEXT +# - name: KAFKA_INTER_BROKER_LISTENER_NAME +# value: LISTENER_PUBLIC diff --git a/examples/test-http.yaml b/examples/test-http.yaml new file mode 100644 index 00000000..035f0ecb --- /dev/null +++ b/examples/test-http.yaml @@ -0,0 +1,36 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hello-app +spec: + selector: + matchLabels: + app: hello + replicas: 1 + template: + metadata: + labels: + app: hello + spec: + containers: + - name: hello + image: "gcr.io/google-samples/hello-app:2.0" + +--- +apiVersion: v1 +kind: Service +metadata: + name: ilb-service + annotations: + networking.gke.io/load-balancer-type: "Internal" + labels: + app: hello +spec: + type: LoadBalancer + selector: + app: hello + ports: + - port: 80 + targetPort: 8080 + protocol: TCP diff --git a/examples/test-networking.yaml b/examples/test-networking.yaml new file mode 100644 index 00000000..e7dbb650 --- /dev/null +++ b/examples/test-networking.yaml @@ -0,0 +1,36 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-networking +spec: + selector: + matchLabels: + app: test-networking + replicas: 1 + template: + metadata: + labels: + app: test-networking + spec: + containers: + - name: ubuntu-test + image: praqma/network-multitool + +--- +apiVersion: v1 +kind: Service +metadata: + name: test-networking-service + annotations: + networking.gke.io/load-balancer-type: "Internal" + labels: + app: test-networking +spec: + type: LoadBalancer + selector: + app: test-networking + ports: + - port: 556 + targetPort: 556 + protocol: UDP diff --git a/examples/trpt-pv.yaml b/examples/trpt-pv.yaml new file mode 100644 index 00000000..f21afb6f --- /dev/null +++ b/examples/trpt-pv.yaml @@ -0,0 +1,15 @@ +kind: PersistentVolume +apiVersion: v1 +metadata: + name: siddhi-pv + labels: + type: local +spec: + storageClassName: standard + persistentVolumeReclaimPolicy: Delete + capacity: + storage: 1Gi + accessModes: + - ReadWriteOnce + hostPath: + path: "/home/siddhi_user/" diff --git a/playbooks/siddhi/kubernetes/trpt-to-kafka.yaml b/examples/trpt-to-kafka.yaml similarity index 78% rename from playbooks/siddhi/kubernetes/trpt-to-kafka.yaml rename to examples/trpt-to-kafka.yaml index 1c240575..134a2b95 100644 --- a/playbooks/siddhi/kubernetes/trpt-to-kafka.yaml +++ b/examples/trpt-to-kafka.yaml @@ -11,6 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +--- apiVersion: siddhi.io/v1alpha2 kind: SiddhiProcess metadata: @@ -35,7 +36,7 @@ spec: @sink( type="kafka", topic="trptPacket", - bootstrap.servers="kafka-service:9092", + bootstrap.servers="10.110.95.121:9092", is.binary.message = "false", @map(type="text") ) @@ -44,7 +45,7 @@ spec: @sink( type="kafka", topic="trptDrop", - bootstrap.servers="kafka-service:9092", + bootstrap.servers="10.110.95.121:9092", is.binary.message = "false", @map(type="text") ) @@ -63,3 +64,23 @@ spec: container: image: "spisarski/siddhi" imagePullPolicy: Always + +--- +apiVersion: v1 +kind: Service +metadata: + name: trpt-to-kafka-0 +spec: + type: LoadBalancer + clusterIP: 10.96.100.2 + externalIPs: + - 192.168.86.181 + selector: + siddhi.io/instance: trpt-to-kafka-0 + siddhi.io/name: SiddhiProcess + siddhi.io/part-of: siddhi-operator + siddhi.io/version: 0.2.2 + ports: + - port: 556 + targetPort: 556 + protocol: UDP From 460a943d201978cdd2f1ae3237a130eddd78baa9 Mon Sep 17 00:00:00 2001 From: spisarski Date: Fri, 30 Jul 2021 15:21:27 -0600 Subject: [PATCH 07/40] improve minikube for siddhi added kafka operator --- playbooks/general/setup_minikube.yml | 2 +- playbooks/general/setup_siddhi_operator.yml | 24 +++++++++++++++------ 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/playbooks/general/setup_minikube.yml b/playbooks/general/setup_minikube.yml index 025382f0..fddc5618 100644 --- a/playbooks/general/setup_minikube.yml +++ b/playbooks/general/setup_minikube.yml @@ -40,7 +40,7 @@ command: snap install kubectl --classic - name: Start minikube to ensure it will work - shell: sudo -u $USER /usr/bin/minikube start + shell: sudo -u $USER /usr/bin/minikube start --memory=4096 register: minkube_start_rc async: 180 retries: 2 diff --git a/playbooks/general/setup_siddhi_operator.yml b/playbooks/general/setup_siddhi_operator.yml index a4ff6fc6..be80e422 100644 --- a/playbooks/general/setup_siddhi_operator.yml +++ b/playbooks/general/setup_siddhi_operator.yml @@ -18,19 +18,13 @@ gather_facts: no tasks: - name: Start minikube to install siddhi-operator - shell: sudo -u $USER /usr/bin/minikube start + shell: sudo -u $USER /usr/bin/minikube start --memory=4096 register: minkube_start_rc async: 180 retries: 2 delay: 5 until: minkube_start_rc is not failed - - name: Install siddhi K8s operator - command: "kubectl apply -f {{ item }}" - loop: - - https://github.com/siddhi-io/siddhi-operator/releases/download/v0.2.2/00-prereqs.yaml - - https://github.com/siddhi-io/siddhi-operator/releases/download/v0.2.2/01-siddhi-operator.yaml - - name: Enable minikube ingress addon command: sudo -u $USER /usr/bin/minikube addons enable ingress register: ing_out @@ -39,6 +33,22 @@ delay: 5 until: ing_out is not failed + - name: Create K8s namespace "kafka" + command: kubectl create namespace kafka + + - name: Create Strimzi Kafka ClusterRoles, ClusterRoleBindings & other CRDs + command: kubectl create -f 'https://strimzi.io/install/latest?namespace=kafka' -n kafka + + - name: Install Siddhi K8s and Strimzi Kafka operator (see https://strimzi.io/quickstarts/) + command: "kubectl apply -f {{ item }}" + loop: + - https://github.com/siddhi-io/siddhi-operator/releases/download/v0.2.2/00-prereqs.yaml + - https://github.com/siddhi-io/siddhi-operator/releases/download/v0.2.2/01-siddhi-operator.yaml + - https://strimzi.io/examples/latest/kafka/kafka-persistent-single.yaml -n kafka + + - name: Wait for Strimzi Kafka operator + command: kubectl wait kafka/my-cluster --for=condition=Ready --timeout=300s -n kafka + - name: Get minikube ip command: sudo -u $USER /usr/bin/minikube ip register: ip_out From bf3cc2d07fb8dbce9f4415b662836a55fef28dc3 Mon Sep 17 00:00:00 2001 From: spisarski Date: Mon, 2 Aug 2021 13:10:03 -0600 Subject: [PATCH 08/40] Able to detect attacks but drop packets are not getting to the AE --- automation/p4/tofino/setup.tf | 4 +- automation/p4/tofino/variables.tf | 3 +- .../scenarios/lab_trial/all-pkt-flood.yml | 27 +++++- playbooks/scenarios/lab_trial/all.yml | 6 +- playbooks/siddhi/docker/Dockerfile | 6 +- .../kafka-trpt-ddos-detection.yaml.j2 | 87 +++++++++++++++++ .../templates/kafka-trpt-drop-clear.yaml.j2 | 84 +++++++++++++++++ .../kubernetes/templates/trpt-pv.yaml.j2 | 15 +++ .../templates/trpt-to-kafka.yaml.j2 | 84 +++++++++++++++++ .../siddhi/templates/minikube_lb.service.j2 | 14 +++ .../templates/minikube_service.service.j2 | 7 +- playbooks/tofino/setup_minikube_siddhi_ae.yml | 94 +++++++++++++++---- playbooks/tofino/setup_nodes-lab_trial.yml | 2 + playbooks/tofino/setup_tofino_switch.yml | 3 +- playbooks/tofino/setup_virt_eth.yml | 5 +- .../topology_template-lab_trial.yaml.j2 | 3 +- playbooks/tofino/tunnel/setup_gre_tunnels.yml | 7 +- 17 files changed, 410 insertions(+), 41 deletions(-) create mode 100644 playbooks/siddhi/kubernetes/templates/kafka-trpt-ddos-detection.yaml.j2 create mode 100644 playbooks/siddhi/kubernetes/templates/kafka-trpt-drop-clear.yaml.j2 create mode 100644 playbooks/siddhi/kubernetes/templates/trpt-pv.yaml.j2 create mode 100644 playbooks/siddhi/kubernetes/templates/trpt-to-kafka.yaml.j2 create mode 100644 playbooks/siddhi/templates/minikube_lb.service.j2 diff --git a/automation/p4/tofino/setup.tf b/automation/p4/tofino/setup.tf index 165d6515..07d5db10 100644 --- a/automation/p4/tofino/setup.tf +++ b/automation/p4/tofino/setup.tf @@ -35,7 +35,8 @@ locals { game_3_ip = var.scenario_name == "full" ? aws_instance.node.6.private_ip: "n/a" # For lab_trial scenario - ae_ip = var.scenario_name == "lab_trial" ? aws_instance.ae.private_ip: "n/a" + ae_ip = var.scenario_name == "lab_trial" ? var.ae_k8s_svc_ip: "n/a" + ae_mgmt_ip = var.scenario_name == "lab_trial" ? aws_instance.ae.private_ip: "n/a" ae_tun1_ip = var.scenario_name == "lab_trial" ? aws_network_interface.ae_tun_1.private_ip: "n/a" ae_tun1_mac = var.scenario_name == "lab_trial" ? aws_network_interface.ae_tun_1.mac_address: "n/a" @@ -165,6 +166,7 @@ inet_ip=${local.lab_inet_ip} inet_tun1_ip=${local.lab_inet_tun1_ip} inet_tun1_mac=${local.lab_inet_tun1_mac} ae_ip=${local.ae_ip} +ae_mgmt_ip=${local.ae_mgmt_ip} ae_tun1_ip=${local.ae_tun1_ip} ae_tun1_mac=${local.ae_tun1_mac} switch_user=${var.switch_user} diff --git a/automation/p4/tofino/variables.tf b/automation/p4/tofino/variables.tf index 367b32b6..33c652a9 100644 --- a/automation/p4/tofino/variables.tf +++ b/automation/p4/tofino/variables.tf @@ -37,7 +37,7 @@ variable "tofino" { } // This is for a Kubernetes Siddhi environment -variable "siddhi_ae_ami" {default = "ami-0b339255a9e7a680e"} +variable "siddhi_ae_ami" {default = "ami-0de45fb321d253856"} // This is a good Maven Siddhi environment //variable "siddhi_ae_ami" {default = "ami-00876f36fe8f97733"} @@ -73,6 +73,7 @@ variable "p4_grpc_port" {default = "50051"} variable "bf_grpc_port" {default = "50052"} variable "p4_bridge_subnet" {default = "192.168.0.0/24"} variable "sdn_port" {default = "9998"} +variable "ae_k8s_svc_ip" {default = "10.96.0.2"} variable "switch_nic_prfx" {default = "veth"} variable "service_log_level" {default = "DEBUG"} variable "ae_monitor_intf" {default = "core-tun"} diff --git a/playbooks/scenarios/lab_trial/all-pkt-flood.yml b/playbooks/scenarios/lab_trial/all-pkt-flood.yml index 4a852131..90c873cb 100644 --- a/playbooks/scenarios/lab_trial/all-pkt-flood.yml +++ b/playbooks/scenarios/lab_trial/all-pkt-flood.yml @@ -16,16 +16,33 @@ # Start AE service - hosts: ae gather_facts: no - become: yes tasks: - name: Start tps-tofino-ae service + become: yes systemd: name: tps-tofino-ae - state: restarted + state: started - - name: Wait 10 seconds for tps-tofino-ae to fully start - pause: - seconds: 10 + - name: Wait for K8s deployment/trpt-to-kafka-0 to become available + command: kubectl wait --for=condition=available --timeout=600s deployment/trpt-to-kafka-0 -n default + register: trpt_to_kafka_check + retries: 10 + delay: 20 + until: trpt_to_kafka_check is not failed + + - name: Wait for K8s deployment/kafka-trpt-ddos-detect-0 to become available + command: kubectl wait --for=condition=available --timeout=600s deployment/kafka-trpt-ddos-detect-0 -n default + register: ddos_check + retries: 5 + delay: 10 + until: ddos_check is not failed + + - name: Wait for K8s deployment/kafka-trpt-drop-clear-0 to become available + command: kubectl wait --for=condition=available --timeout=600s deployment/kafka-trpt-drop-clear-0 -n default + register: drop_check + retries: 5 + delay: 10 + until: drop_check is not failed # Packet Flood scenarios for UDP and IPv4 - import_playbook: pkt-flood.yml diff --git a/playbooks/scenarios/lab_trial/all.yml b/playbooks/scenarios/lab_trial/all.yml index c9ec0871..677630e4 100644 --- a/playbooks/scenarios/lab_trial/all.yml +++ b/playbooks/scenarios/lab_trial/all.yml @@ -27,11 +27,11 @@ # Data Drop scenarios - import_playbook: all-data-drop.yml +# Switches now operational - Run Packet Flood scenarios to test the AE +- import_playbook: all-pkt-flood.yml + # Drop Reporting scenarios - import_playbook: all-drop-rpt.yml # Packet Performance tests - import_playbook: iperf.yml - -# Switches now operational - Run Packet Flood scenarios to test the AE -- import_playbook: all-pkt-flood.yml diff --git a/playbooks/siddhi/docker/Dockerfile b/playbooks/siddhi/docker/Dockerfile index fd7e412f..57fb1c98 100644 --- a/playbooks/siddhi/docker/Dockerfile +++ b/playbooks/siddhi/docker/Dockerfile @@ -15,11 +15,7 @@ FROM siddhiio/siddhi-runner-alpine:5.1.2 RUN apk --update add net-tools tcpdump sudo RUN apk update && apk add ca-certificates && rm -rf /var/cache/apk/* -#RUN set -ex && apk --no-cache add sudo - -#SHELL ["/bin/bash", "-o", "pipefail", "-c"] -#RUN addgroup siddhi_user sudo -RUN echo 'siddhi_user:siddhi_user' | chpasswd +RUN echo 'siddhi_user:siddhi' | chpasswd RUN echo '%wheel ALL=(ALL) ALL' > /etc/sudoers.d/wheel RUN adduser siddhi_user wheel diff --git a/playbooks/siddhi/kubernetes/templates/kafka-trpt-ddos-detection.yaml.j2 b/playbooks/siddhi/kubernetes/templates/kafka-trpt-ddos-detection.yaml.j2 new file mode 100644 index 00000000..a8dcb0b1 --- /dev/null +++ b/playbooks/siddhi/kubernetes/templates/kafka-trpt-ddos-detection.yaml.j2 @@ -0,0 +1,87 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: siddhi.io/v1alpha2 +kind: SiddhiProcess +metadata: + name: kafka-trpt-ddos-detect +spec: + apps: + - script: | + @App:name("KafkaSourcePacketJSON") + @source( + type="kafka", + topic.list="trptPacket", + bootstrap.servers="{{ kafka_svc_ip }}:9092", + group.id="test", + threading.option="single.thread", + @map( + type="p4-trpt", + @attributes( + src_mac="intHdr.mdStackHdr.origMac", + ip_ver="ipHdr.version", + dst_ip="ipHdr.dstAddr", + dst_port="protoHdr.dstPort" + ) + ) + ) + define stream trptPktStream ( + src_mac string, + ip_ver int, + dst_ip string, + dst_port long); + + @sink( + type="http", + publisher.url="http://{{ sdn_ip }}:9998/aggAttack", + method="POST", + headers="trp:headers", + @map(type="json") + ) + define stream attackStream ( + src_mac string, + ip_ver int, + dst_ip string, + dst_port long, + count long + ); + + @info(name = "trptJsonQuery") + from trptPktStream#window.time(1 sec) + select src_mac, ip_ver, dst_ip, dst_port, count(ip_ver) as count + group by src_mac, dst_ip, dst_port + having count == 10 + insert into attackStream; + + container: + image: "spisarski/siddhi" + imagePullPolicy: Always + + persistentVolumeClaim: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: standard + volumeMode: Filesystem + + runner: | + state.persistence: + enabled: true + intervalInMin: 5 + revisionsToKeep: 2 + persistenceStore: io.siddhi.distribution.core.persistence.FileSystemPersistenceStore + config: + location: siddhi-app-persistence diff --git a/playbooks/siddhi/kubernetes/templates/kafka-trpt-drop-clear.yaml.j2 b/playbooks/siddhi/kubernetes/templates/kafka-trpt-drop-clear.yaml.j2 new file mode 100644 index 00000000..3f80d227 --- /dev/null +++ b/playbooks/siddhi/kubernetes/templates/kafka-trpt-drop-clear.yaml.j2 @@ -0,0 +1,84 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: siddhi.io/v1alpha2 +kind: SiddhiProcess +metadata: + name: kafka-trpt-drop-clear +spec: + apps: + - script: | + @App:name("KafkaSourceDropJSON") + @source( + type="kafka", + topic.list="trptDrop", + bootstrap.servers="{{ kafka_svc_ip }}:9092", + group.id="test", + threading.option="single.thread", + @map( + type="p4-trpt", + @attributes( + timestamp="dropHdr.timestamp", + dropKey="dropHdr.dropKey", + dropCount="dropHdr.dropCount" + ) + ) + ) + define stream trptDropStream ( + timestamp long, + dropKey string, + dropCount long + ); + + @sink( + type="http", + publisher.url="http://{{ sdn_ip }}:9998/aggAttack", + method="DELETE", + headers="trp:headers", + @map(type="json") + ) + define stream dropAttackStream ( + dropKey string, + dropCount long, + count long + ); + + @info(name = "trptJsonQuery") + from trptDropStream#window.time(35 sec) + select dropKey, dropCount, count(dropCount) as count + group by dropKey, dropCount + having count >= 3 + insert into dropAttackStream; + + container: + image: "spisarski/siddhi" + imagePullPolicy: Always + + persistentVolumeClaim: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: standard + volumeMode: Filesystem + + runner: | + state.persistence: + enabled: true + intervalInMin: 5 + revisionsToKeep: 2 + persistenceStore: io.siddhi.distribution.core.persistence.FileSystemPersistenceStore + config: + location: siddhi-app-persistence diff --git a/playbooks/siddhi/kubernetes/templates/trpt-pv.yaml.j2 b/playbooks/siddhi/kubernetes/templates/trpt-pv.yaml.j2 new file mode 100644 index 00000000..f21afb6f --- /dev/null +++ b/playbooks/siddhi/kubernetes/templates/trpt-pv.yaml.j2 @@ -0,0 +1,15 @@ +kind: PersistentVolume +apiVersion: v1 +metadata: + name: siddhi-pv + labels: + type: local +spec: + storageClassName: standard + persistentVolumeReclaimPolicy: Delete + capacity: + storage: 1Gi + accessModes: + - ReadWriteOnce + hostPath: + path: "/home/siddhi_user/" diff --git a/playbooks/siddhi/kubernetes/templates/trpt-to-kafka.yaml.j2 b/playbooks/siddhi/kubernetes/templates/trpt-to-kafka.yaml.j2 new file mode 100644 index 00000000..8bb5a190 --- /dev/null +++ b/playbooks/siddhi/kubernetes/templates/trpt-to-kafka.yaml.j2 @@ -0,0 +1,84 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: siddhi.io/v1alpha2 +kind: SiddhiProcess +metadata: + name: trpt-to-kafka +spec: + apps: + - script: | + @App:name("UDPSourceTRPT") + @source( + type="udp", + listen.port="556", + @map( + type="p4-trpt", + @attributes( + in_type="telemRptHdr.inType", + full_json="jsonString" + ) + ) + ) + define stream trptUdpStream (in_type int, full_json object); + + @sink( + type="kafka", + topic="trptPacket", + bootstrap.servers="{{ kafka_svc_ip }}:9092", + is.binary.message = "false", + @map(type="text") + ) + define stream trptPacket (full_json object); + + @sink( + type="kafka", + topic="trptDrop", + bootstrap.servers="{{ kafka_svc_ip }}:9092", + is.binary.message = "false", + @map(type="text") + ) + define stream trptDrop (full_json object); + + @info(name = "TrptPacket") + from trptUdpStream[in_type != 2] + select full_json + insert into trptPacket; + + @info(name = "TrptDrop") + from trptUdpStream[in_type == 2] + select full_json + insert into trptDrop; + + container: + image: "spisarski/siddhi" + imagePullPolicy: Always + +--- +apiVersion: v1 +kind: Service +metadata: + name: trpt-to-kafka-0 +spec: + type: LoadBalancer + clusterIP: {{ ae_ip }} + selector: + siddhi.io/instance: trpt-to-kafka-0 + siddhi.io/name: SiddhiProcess + siddhi.io/part-of: siddhi-operator + siddhi.io/version: 0.2.2 + ports: + - port: 556 + targetPort: 556 + protocol: UDP diff --git a/playbooks/siddhi/templates/minikube_lb.service.j2 b/playbooks/siddhi/templates/minikube_lb.service.j2 new file mode 100644 index 00000000..bc3ba16d --- /dev/null +++ b/playbooks/siddhi/templates/minikube_lb.service.j2 @@ -0,0 +1,14 @@ +[Unit] +Description=Minikube service for TPS AE +Wants=minikube.service +Requisite={{ minikube_srvc }} +After={{ minikube_srvc }} +PartOf={{ minikube_srvc }} + +[Service] +User=ubuntu +#Group=docker +ExecStart=/usr/bin/minikube tunnel +ExecStop=/usr/bin/systemctl stop minikube.service +[Install] +WantedBy=multi-user.target diff --git a/playbooks/siddhi/templates/minikube_service.service.j2 b/playbooks/siddhi/templates/minikube_service.service.j2 index 17b89ea9..f09fb2d9 100644 --- a/playbooks/siddhi/templates/minikube_service.service.j2 +++ b/playbooks/siddhi/templates/minikube_service.service.j2 @@ -1,14 +1,15 @@ [Unit] Description=Minikube service for TPS AE -Requires=network.target -After=syslog.target network.target +Requires=network.target docker.service +After=syslog.target network.target docker.service +PartOf={{ part_of_srvc }} [Service] Type=oneshot RemainAfterExit=yes User=ubuntu Group=docker -ExecStart=/usr/bin/minikube start +ExecStart=/usr/bin/minikube start --memory=4096 ExecStop=/usr/bin/minikube stop [Install] diff --git a/playbooks/tofino/setup_minikube_siddhi_ae.yml b/playbooks/tofino/setup_minikube_siddhi_ae.yml index f3b35832..881034d6 100644 --- a/playbooks/tofino/setup_minikube_siddhi_ae.yml +++ b/playbooks/tofino/setup_minikube_siddhi_ae.yml @@ -15,13 +15,23 @@ - hosts: "{{ host_val | default('all') }}" gather_facts: no tasks: - - name: Create Service File to /etc/systemd/system/tps-tofino-ae.service + - name: Create Service File to /etc/systemd/system/minikube.service become: yes template: src: ../siddhi/templates/minikube_service.service.j2 + dest: /etc/systemd/system/minikube.service + vars: + part_of_srvc: tps-tofino-ae.service + + - name: Create Service File to /etc/systemd/system/tps-tofino-ae.service + become: yes + template: + src: ../siddhi/templates/minikube_lb.service.j2 dest: /etc/systemd/system/tps-tofino-ae.service + vars: + minikube_srvc: minikube.service - - name: Start tps-tofino-ae.service + - name: Start TPS AE become: yes systemd: name: tps-tofino-ae.service @@ -43,28 +53,80 @@ debug: var: kube_pods_out.stdout_lines - - name: Install siddhi K8s operator - command: "kubectl apply -f {{ item }}" - loop: - - "{{ trans_sec_dir }}/playbooks/siddhi/kubernetes/kafka.yaml" - - "{{ trans_sec_dir }}/playbooks/siddhi/kubernetes/trpt-to-kafka.yaml" - - "{{ trans_sec_dir }}/playbooks/siddhi/kubernetes/kafka-trpt-ddos-detection.yaml" - - "{{ trans_sec_dir }}/playbooks/siddhi/kubernetes/kafka-trpt-drop-clear.yaml" + - name: Ensure {{ remote_scripts_dir }} scripts directory has been created + become: yes + file: + path: "{{ remote_scripts_dir }}" + state: directory + + - name: Query the Kafka Service clusterIP + command: kubectl get svc -n kafka my-cluster-kafka-bootstrap -o jsonpath="{.spec.clusterIPs[0]}" + register: cluster_ip_out - # TODO - Improve validation by for all containing "Running" - - name: Wait for K8s pods to come up... - command: kubectl get po - register: kube_pods_out + - name: Define the K8s CRDs + set_fact: + the_crds: + - "kafka-trpt-ddos-detection.yaml" + - "kafka-trpt-drop-clear.yaml" + - "trpt-to-kafka.yaml" + kafka_svc_ip: "{{ cluster_ip_out.stdout_lines[0] }}" + + - debug: + var: the_crds + + - name: Applying J2 templates of the siddhi K8s CRDs to {{ remote_scripts_dir }} with Kafka service IP - {{ kafka_svc_ip }} + become: yes + template: + src: "{{ trans_sec_dir }}/playbooks/siddhi/kubernetes/templates/{{ item }}.j2" + dest: "{{ remote_scripts_dir }}/{{ item }}" + loop: "{{ the_crds }}" + + - name: Install siddhi K8s SiddhiProcess + command: "kubectl apply -f {{ remote_scripts_dir }}/{{ item }}" + loop: "{{ the_crds }}" + + - name: Wait for K8s deployment/trpt-to-kafka-0 to become available + command: kubectl wait --for=condition=available --timeout=600s deployment/trpt-to-kafka-0 -n default + register: trpt_to_kafka_check retries: 10 + delay: 20 + until: trpt_to_kafka_check is not failed + + - name: Wait for K8s deployment/kafka-trpt-ddos-detect-0 to become available + command: kubectl wait --for=condition=available --timeout=600s deployment/kafka-trpt-ddos-detect-0 -n default + register: ddos_check + retries: 5 + delay: 10 + until: ddos_check is not failed + + - name: Wait for K8s deployment/kafka-trpt-drop-clear-0 to become available + command: kubectl wait --for=condition=available --timeout=600s deployment/kafka-trpt-drop-clear-0 -n default + register: drop_check + retries: 5 delay: 10 - until: kube_pods_out.stdout.find("Running") != -1 and - kube_pods_out.stdout.find("Creating") == -1 and - kube_pods_out.stdout_lines | length >= 9 + until: drop_check is not failed + + - name: Get pods in default namespace + command: kubectl get pods -n default + register: kube_pods_out - name: Show pods debug: var: kube_pods_out.stdout_lines + # TODO - Find a better means to obtain this bridge as things may break as soon as another interface prefaced with "br-" is created + - name: Query for the docker/K8s bridge + shell: "ip -o link show | awk -F': ' '{print $2}'| grep br-" + register: bridge_name_out + + - name: Store the CRDs + set_fact: + docker_bridge_name: "{{ bridge_name_out.stdout_lines[0] }}" + + # TODO - Reconsider this approach which was hacked in place as TRPTs aren't getting routed into the pod + - name: Forward all {{ clone_tun_name }} packets to the Docker/K8s service bridge {{ docker_bridge_name }} + command: "sudo iptables -A FORWARD -i {{ clone_tun_name }} -o {{ docker_bridge_name }} -j ACCEPT" + - name: Stop tps-tofino-ae.service become: yes systemd: diff --git a/playbooks/tofino/setup_nodes-lab_trial.yml b/playbooks/tofino/setup_nodes-lab_trial.yml index 7f343883..539cc9be 100644 --- a/playbooks/tofino/setup_nodes-lab_trial.yml +++ b/playbooks/tofino/setup_nodes-lab_trial.yml @@ -24,6 +24,7 @@ - import_playbook: setup_minikube_siddhi_ae.yml vars: host_val: ae + clone_tun_name: core-tun1 # Create virtual interfaces on switches - import_playbook: setup_virt_eth.yml @@ -51,3 +52,4 @@ port_to_wait: "{{ sdn_port }}" wait_timeout: 60 load_p4: False + diff --git a/playbooks/tofino/setup_tofino_switch.yml b/playbooks/tofino/setup_tofino_switch.yml index 669cf892..efd3dd5d 100644 --- a/playbooks/tofino/setup_tofino_switch.yml +++ b/playbooks/tofino/setup_tofino_switch.yml @@ -27,6 +27,7 @@ # Start tofino model chip emulator - import_playbook: ../general/start_service.yml vars: + setup_for_hw: "{{ from_hw | default(False) }}" host_val: "{{ switches }}" service_name: tps-tofino-model topo_dict: "{{ lookup('file','{{ topo_file_loc }}') | from_yaml }}" @@ -37,7 +38,7 @@ prog_name: "{{ p4_prog }}" additional_tmplt_file: "{{ trans_sec_dir }}/playbooks/tofino/templates/tofino-model-veth-port-mapping.json.j2" additional_tmplt_out_file: "{{ remote_scripts_dir }}/port-mapping.json" - when: not from_hw | bool + when: not setup_for_hw | bool # Start switchd - import_playbook: ../general/start_service.yml diff --git a/playbooks/tofino/setup_virt_eth.yml b/playbooks/tofino/setup_virt_eth.yml index a318d7ab..363428dc 100644 --- a/playbooks/tofino/setup_virt_eth.yml +++ b/playbooks/tofino/setup_virt_eth.yml @@ -20,10 +20,11 @@ SDE: "{{ remote_sde_dir }}" SDE_INSTALL: "{{ remote_sde_dir }}/install" vars: - setup_veth: "{{ not from_hw|bool | default(True) }}" + setup_for_hw: "{{ from_hw | default(False) }}" + setup_veth: "{{ not setup_for_hw | bool }}" tasks: - debug: - var: from_hw + var: setup_for_hw - debug: var: setup_veth - name: Setup DMA diff --git a/playbooks/tofino/templates/topology_template-lab_trial.yaml.j2 b/playbooks/tofino/templates/topology_template-lab_trial.yaml.j2 index a0c356a4..be25796e 100644 --- a/playbooks/tofino/templates/topology_template-lab_trial.yaml.j2 +++ b/playbooks/tofino/templates/topology_template-lab_trial.yaml.j2 @@ -73,7 +73,8 @@ hosts: {% if host_user is defined %} user: {{ host_user }} {% endif %} - public_ip: {{ ae_ip }} + public_ip: {{ ae_mgmt_ip }} + streaming_ip: {{ ae_ip }} tun1_ip: {{ ae_tun1_ip }} tun1_mac: {{ ae_tun1_mac }} tunnels: diff --git a/playbooks/tofino/tunnel/setup_gre_tunnels.yml b/playbooks/tofino/tunnel/setup_gre_tunnels.yml index b1190e6a..e326e644 100644 --- a/playbooks/tofino/tunnel/setup_gre_tunnels.yml +++ b/playbooks/tofino/tunnel/setup_gre_tunnels.yml @@ -19,10 +19,11 @@ gather_facts: yes become: yes vars: - setup_tunnels: "{{ not from_hw|bool | default(True) }}" + setup_for_hw: "{{ from_hw | default(False) }}" + setup_tunnels: "{{ not setup_for_hw | bool }}" tasks: - debug: - var: from_hw + var: setup_for_hw - debug: var: setup_tunnels - block: @@ -38,7 +39,7 @@ name: ip_gre state: present - - name: install ip_gre module + - name: Setup ipv4 forwarding lineinfile: path: /etc/sysctl.conf line: "net.ipv4.ip_forward=1" From c6563c4f38b0148d2d0452c539a2e51eb2cfbca9 Mon Sep 17 00:00:00 2001 From: spisarski Date: Wed, 4 Aug 2021 13:32:01 -0600 Subject: [PATCH 09/40] Added MD for running the Siddhi AE and renamed the snaps-hcp AE directory from "analytics" to "snaps-hcp" --- automation/p4/tofino/variables.tf | 5 ++- docs/SIDDHI_AE_SETUP.md | 37 +++++++++++++++++++ .../OpenDistroElasticSearch/Readme.md | 0 .../anomaly_detector_config.json | 0 .../OpenDistroElasticSearch/data_mapping.json | 0 .../ddos_detection_and_mitigation_pipeline.md | 0 .../packet_template-7.x.sh | 0 .../udp_data_mapping_and_parsing_pipeline.md | 0 .../udp_data_parsing.json | 0 .../udp_data_parsing_pipeline.sh | 0 {analytics => snaps-hcp}/Readme.md | 7 +++- {analytics => snaps-hcp}/setup/INSTALL.md | 0 12 files changed, 46 insertions(+), 3 deletions(-) create mode 100644 docs/SIDDHI_AE_SETUP.md rename {analytics => snaps-hcp}/OpenDistroElasticSearch/Readme.md (100%) rename {analytics => snaps-hcp}/OpenDistroElasticSearch/anomaly_detector_config.json (100%) rename {analytics => snaps-hcp}/OpenDistroElasticSearch/data_mapping.json (100%) rename {analytics => snaps-hcp}/OpenDistroElasticSearch/ddos_detection_and_mitigation_pipeline.md (100%) rename {analytics => snaps-hcp}/OpenDistroElasticSearch/packet_template-7.x.sh (100%) rename {analytics => snaps-hcp}/OpenDistroElasticSearch/udp_data_mapping_and_parsing_pipeline.md (100%) rename {analytics => snaps-hcp}/OpenDistroElasticSearch/udp_data_parsing.json (100%) rename {analytics => snaps-hcp}/OpenDistroElasticSearch/udp_data_parsing_pipeline.sh (100%) rename {analytics => snaps-hcp}/Readme.md (92%) rename {analytics => snaps-hcp}/setup/INSTALL.md (100%) diff --git a/automation/p4/tofino/variables.tf b/automation/p4/tofino/variables.tf index 6d6472e5..c82b9fa1 100644 --- a/automation/p4/tofino/variables.tf +++ b/automation/p4/tofino/variables.tf @@ -36,7 +36,10 @@ variable "tofino" { } } -variable "siddhi_ae_ami" {default = "ami-00876f36fe8f97733"} +variable "siddhi_ae_ami" {default = "ami-08ec1da30d3413105"} +//variable "siddhi_ae_ami" {default = "ami-0e7d14e85b77459c4"} +//variable "siddhi_ae_ami" {default = "ami-06ea3285e31e16516"} +//variable "siddhi_ae_ami" {default = "ami-00876f36fe8f97733"} variable "switch_instance_type" {default = "t2.2xlarge"} variable "ae_instance_type" {default = "t2.2xlarge"} diff --git a/docs/SIDDHI_AE_SETUP.md b/docs/SIDDHI_AE_SETUP.md new file mode 100644 index 00000000..12b1385a --- /dev/null +++ b/docs/SIDDHI_AE_SETUP.md @@ -0,0 +1,37 @@ +# TPS Siddhi AE + +The Transparent Security . + +1. Open source python program Espcap for live data capture. ( https://github.com/vichargrave/espcap ) +2. Open source distribution of ElasticSearch. ( https://opendistro.github.io/for-elasticsearch-docs/ ) + +## Host Software Requirements +1. Java 8+ +1. git +1. maven +1. kafka + +## Setup and run +There are several things that must be done to the host prior to attempting to start +the Siddhi TPS AE running as a standalone Java application + +- Install Java 8+ (CI is running OpenJDK 11.0.11) +- Install Git client (apt install git) +- Install Maven client (apt install maven) +- Install and start Kafka to port 9092 (see https://www.digitalocean.com/community/tutorials/how-to-install-apache-kafka-on-ubuntu-20-04) +- Clone the following Git project: https://github.com/cablelabs/siddhi-map-p4-trpt.git +- Open shell to the cloned directory (e.g. cd ~/siddhi-map-p4-trpt) +- Execute +```bash +mvn exec:java -Dexec.mainClass=io.siddhi.extension.map.p4.StartSiddhiRuntime \ +-Dexec.args="/etc/transparent-security/convert_trpt.siddhi \ +/etc/transparent-security/simple_ddos_detection.siddhi \ +/etc/transparent-security/simple_ddos_clear.siddhi \ +" -f pom.xml +``` + +## Making changes +The StartSiddhiRuntime Java main() simply starts a Siddhi Runtime then takes +the file contents of each of the arguments and loads them into the engine. +Therefore, changes can be made to any or all of the siddhi files and/or other +Siddhi scripts can be added/deleted. diff --git a/analytics/OpenDistroElasticSearch/Readme.md b/snaps-hcp/OpenDistroElasticSearch/Readme.md similarity index 100% rename from analytics/OpenDistroElasticSearch/Readme.md rename to snaps-hcp/OpenDistroElasticSearch/Readme.md diff --git a/analytics/OpenDistroElasticSearch/anomaly_detector_config.json b/snaps-hcp/OpenDistroElasticSearch/anomaly_detector_config.json similarity index 100% rename from analytics/OpenDistroElasticSearch/anomaly_detector_config.json rename to snaps-hcp/OpenDistroElasticSearch/anomaly_detector_config.json diff --git a/analytics/OpenDistroElasticSearch/data_mapping.json b/snaps-hcp/OpenDistroElasticSearch/data_mapping.json similarity index 100% rename from analytics/OpenDistroElasticSearch/data_mapping.json rename to snaps-hcp/OpenDistroElasticSearch/data_mapping.json diff --git a/analytics/OpenDistroElasticSearch/ddos_detection_and_mitigation_pipeline.md b/snaps-hcp/OpenDistroElasticSearch/ddos_detection_and_mitigation_pipeline.md similarity index 100% rename from analytics/OpenDistroElasticSearch/ddos_detection_and_mitigation_pipeline.md rename to snaps-hcp/OpenDistroElasticSearch/ddos_detection_and_mitigation_pipeline.md diff --git a/analytics/OpenDistroElasticSearch/packet_template-7.x.sh b/snaps-hcp/OpenDistroElasticSearch/packet_template-7.x.sh similarity index 100% rename from analytics/OpenDistroElasticSearch/packet_template-7.x.sh rename to snaps-hcp/OpenDistroElasticSearch/packet_template-7.x.sh diff --git a/analytics/OpenDistroElasticSearch/udp_data_mapping_and_parsing_pipeline.md b/snaps-hcp/OpenDistroElasticSearch/udp_data_mapping_and_parsing_pipeline.md similarity index 100% rename from analytics/OpenDistroElasticSearch/udp_data_mapping_and_parsing_pipeline.md rename to snaps-hcp/OpenDistroElasticSearch/udp_data_mapping_and_parsing_pipeline.md diff --git a/analytics/OpenDistroElasticSearch/udp_data_parsing.json b/snaps-hcp/OpenDistroElasticSearch/udp_data_parsing.json similarity index 100% rename from analytics/OpenDistroElasticSearch/udp_data_parsing.json rename to snaps-hcp/OpenDistroElasticSearch/udp_data_parsing.json diff --git a/analytics/OpenDistroElasticSearch/udp_data_parsing_pipeline.sh b/snaps-hcp/OpenDistroElasticSearch/udp_data_parsing_pipeline.sh similarity index 100% rename from analytics/OpenDistroElasticSearch/udp_data_parsing_pipeline.sh rename to snaps-hcp/OpenDistroElasticSearch/udp_data_parsing_pipeline.sh diff --git a/analytics/Readme.md b/snaps-hcp/Readme.md similarity index 92% rename from analytics/Readme.md rename to snaps-hcp/Readme.md index 96d62e14..5096c78b 100644 --- a/analytics/Readme.md +++ b/snaps-hcp/Readme.md @@ -1,6 +1,9 @@ -# Analytics Guide +# snaps-hcp Analytics Guide -The transparent security analytics module uses the following components for data capture and data analysis. +(deprecated) +The files contained within these directories are requried for setting up the +SNAPS Hortonworks analytics platform (aka. snap-hcp) which has been replaced +by a Siddhi engine and scripts (see docs/SIDDHI_AE_SETUP.md). 1. Open source python program Espcap for live data capture. ( https://github.com/vichargrave/espcap ) 2. Open source distribution of ElasticSearch. ( https://opendistro.github.io/for-elasticsearch-docs/ ) diff --git a/analytics/setup/INSTALL.md b/snaps-hcp/setup/INSTALL.md similarity index 100% rename from analytics/setup/INSTALL.md rename to snaps-hcp/setup/INSTALL.md From 4fc4f3b4ccd778b95d287b5a77077db6667548b0 Mon Sep 17 00:00:00 2001 From: spisarski Date: Wed, 4 Aug 2021 13:51:07 -0600 Subject: [PATCH 10/40] Added the top paragraph I totally spaced. --- docs/SIDDHI_AE_SETUP.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/SIDDHI_AE_SETUP.md b/docs/SIDDHI_AE_SETUP.md index 12b1385a..20d49a5b 100644 --- a/docs/SIDDHI_AE_SETUP.md +++ b/docs/SIDDHI_AE_SETUP.md @@ -1,9 +1,8 @@ # TPS Siddhi AE -The Transparent Security . - -1. Open source python program Espcap for live data capture. ( https://github.com/vichargrave/espcap ) -2. Open source distribution of ElasticSearch. ( https://opendistro.github.io/for-elasticsearch-docs/ ) +The Transparent Security TPS AE is currently being run as a simple Java main +program within our "lab_trial" scenario tests. This program's only responsibility +is to start a Siddhi runtime and load Siddhi scripts. ## Host Software Requirements 1. Java 8+ From 15e18970efe876c66e8be34b45b717094abc730b Mon Sep 17 00:00:00 2001 From: spisarski Date: Wed, 4 Aug 2021 15:21:06 -0600 Subject: [PATCH 11/40] Added short description of each top-level directory --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.md b/README.md index 1167a20b..d4343754 100644 --- a/README.md +++ b/README.md @@ -31,3 +31,14 @@ We use an [Apache 2.0 License](LICENSE) for Transparent Security. Questions? Just send us an email at [transparent-security@cablelabs.com](mailto:transparent-security@cablelabs.com) or [open an issue](https://github.com/cablelabs/transparent-security/issues). + +## The directories +- automation - contains Terraform scripts for CI and testing on AWS +- bin - miscellaneous scripts mostly used by scripts in automation +- conf - miscellaneous environment configurations +- docs - miscellaneous MD files +- p4 - The P4 source code +- playbooks - The Ansible Playbooks used by automation +- snaps-hcp - Documentation on snaps-hcp which is no longer being used +- tests - the Python unit test directory +- trans_sec - the project's top-level Python package \ No newline at end of file From 1761fde550b08b9f2a8e73594ffd146633c6c461 Mon Sep 17 00:00:00 2001 From: spisarski Date: Wed, 4 Aug 2021 15:35:43 -0600 Subject: [PATCH 12/40] Added Randy's suggestion from the MR comment and clarified to the Java main kickoff section below the new addition --- docs/SIDDHI_AE_SETUP.md | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/docs/SIDDHI_AE_SETUP.md b/docs/SIDDHI_AE_SETUP.md index 20d49a5b..1ac789a1 100644 --- a/docs/SIDDHI_AE_SETUP.md +++ b/docs/SIDDHI_AE_SETUP.md @@ -1,26 +1,39 @@ # TPS Siddhi AE -The Transparent Security TPS AE is currently being run as a simple Java main -program within our "lab_trial" scenario tests. This program's only responsibility -is to start a Siddhi runtime and load Siddhi scripts. +The Transparent Security TPS AE is an implementation of transparent security +analytics engine based on Siddhi. It is able to identify a DDoS attack in about +1 second by examining telemetry reports generated by a switch or router. This +will also identify when a mitigated attack has based on drop telemetry reports +from the SDN controller. + +The directions below are how to run this as a simple Java program (as with our +"lab_trial" scenario tests), but these scripts can also be deployed to any +Siddhi compliant engine containing our UDP & P4-TRPT extensions (and a running +Kafka instance). Additionally, Siddhi scripts can also be converted to run in +Kubernetes where the siddhi-operator has been installed. ## Host Software Requirements + 1. Java 8+ 1. git 1. maven 1. kafka -## Setup and run -There are several things that must be done to the host prior to attempting to start -the Siddhi TPS AE running as a standalone Java application +## Setup and run + +There are several things that must be done to the host prior to attempting to +start the Siddhi TPS AE running as a standalone Java application - Install Java 8+ (CI is running OpenJDK 11.0.11) - Install Git client (apt install git) - Install Maven client (apt install maven) -- Install and start Kafka to port 9092 (see https://www.digitalocean.com/community/tutorials/how-to-install-apache-kafka-on-ubuntu-20-04) -- Clone the following Git project: https://github.com/cablelabs/siddhi-map-p4-trpt.git +- Install and start Kafka to port 9092 ( + see https://www.digitalocean.com/community/tutorials/how-to-install-apache-kafka-on-ubuntu-20-04) +- Clone the following Git + project: https://github.com/cablelabs/siddhi-map-p4-trpt.git - Open shell to the cloned directory (e.g. cd ~/siddhi-map-p4-trpt) - Execute + ```bash mvn exec:java -Dexec.mainClass=io.siddhi.extension.map.p4.StartSiddhiRuntime \ -Dexec.args="/etc/transparent-security/convert_trpt.siddhi \ @@ -30,6 +43,7 @@ mvn exec:java -Dexec.mainClass=io.siddhi.extension.map.p4.StartSiddhiRuntime \ ``` ## Making changes + The StartSiddhiRuntime Java main() simply starts a Siddhi Runtime then takes the file contents of each of the arguments and loads them into the engine. Therefore, changes can be made to any or all of the siddhi files and/or other From d1b626bb2bfde2badbc7679b09e994c0256fc24e Mon Sep 17 00:00:00 2001 From: Randy Levensalor Date: Wed, 4 Aug 2021 17:37:41 -0600 Subject: [PATCH 13/40] Update SIDDHI_AE_SETUP.md Fixed a typeo --- docs/SIDDHI_AE_SETUP.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/SIDDHI_AE_SETUP.md b/docs/SIDDHI_AE_SETUP.md index 1ac789a1..26ba879f 100644 --- a/docs/SIDDHI_AE_SETUP.md +++ b/docs/SIDDHI_AE_SETUP.md @@ -3,7 +3,7 @@ The Transparent Security TPS AE is an implementation of transparent security analytics engine based on Siddhi. It is able to identify a DDoS attack in about 1 second by examining telemetry reports generated by a switch or router. This -will also identify when a mitigated attack has based on drop telemetry reports +will also identify when a mitigated attack has ended based on drop telemetry reports from the SDN controller. The directions below are how to run this as a simple Java program (as with our From 0aab0bc85979293bbea9f48aa1bf2375135f0420 Mon Sep 17 00:00:00 2001 From: spisarski Date: Fri, 6 Aug 2021 09:21:12 -0600 Subject: [PATCH 14/40] Initial mass cleanup of old files that no longer work. See siddhi-ae tag for files prior to this. --- README.md | 2 +- automation/env-build/aws-inst.tf | 5 - automation/env-build/variables.tf | 10 +- automation/p4/mininet/aws-inst.tf | 74 - automation/p4/mininet/aws-security.tf | 89 - automation/p4/mininet/outputs.tf | 25 - automation/p4/mininet/scenarios.tf | 33 - automation/p4/mininet/setup.tf | 150 -- automation/p4/mininet/variables.tf | 63 - automation/p4/tofino/aws-insts.tf | 24 +- automation/p4/tofino/setup.tf | 54 +- automation/p4/tofino/variables.tf | 10 +- bin/run_p4_mininet.py | 84 - bin/sdn_controller.py | 18 +- bin/start_ae.py | 117 -- docs/BUILD.md | 465 ----- docs/{ => ae}/SIDDHI_AE_SETUP.md | 0 docs/mininet-example.tfvars | 27 - p4/aggregate/aggregate.p4 | 273 --- p4/core/core.p4 | 336 ---- p4/gateway/gateway.p4 | 360 ---- playbooks/dependencies/bmv2.yml | 155 -- playbooks/dependencies/grpc.yml | 77 - playbooks/dependencies/mininet.yml | 50 - playbooks/dependencies/p4c.yml | 88 - playbooks/dependencies/p4runtime.yml | 85 - playbooks/dependencies/protobuf.yml | 91 - playbooks/env-build/ae/env_build.yml | 17 - playbooks/env-build/mininet/env_build.yml | 23 - playbooks/general/final_env_setup.yml | 2 - .../general/templates/mininet_service.sh.j2 | 17 - .../general/templates/sdn_controller.sh.j2 | 2 +- playbooks/hcp/ae_node_build.yml | 15 - playbooks/hcp/conf_ae_elk.yml | 39 - playbooks/hcp/delete_ae_history.yml | 27 - playbooks/hcp/setup_ae_centos.yml | 242 --- playbooks/hcp/setup_ae_tcp_pipeline.yml | 165 -- .../hcp/setup_ae_udp_and_tcp_pipeline.yml | 195 -- playbooks/hcp/setup_ae_udp_pipeline.yml | 162 -- playbooks/hcp/start_ae_elk.yml | 40 - .../templates/drop_report_hash_script.json | 158 -- .../templates/index_pattern_request_body.json | 6 - .../hcp/templates/opendistro_elasticsearch.sh | 113 -- playbooks/hcp/templates/opendistro_kibana.sh | 31 - playbooks/hcp/templates/query_monitor_id.json | 7 - .../hcp/templates/sdn_attack_webhook.json | 13 - .../hcp/templates/tcp/tcp_data_mapping.json | 292 --- .../hcp/templates/tcp/tcp_data_parsing.json | 1332 ------------ .../hcp/templates/tcp/tcp_monitor_def.json | 156 -- .../hcp/templates/udp/udp_data_mapping.json | 315 --- .../hcp/templates/udp/udp_data_parsing.json | 1536 -------------- .../hcp/templates/udp/udp_monitor_def.json | 166 -- .../templates/udp_and_tcp_data_mapping.json | 320 --- .../templates/udp_and_tcp_data_parsing.json | 1777 ----------------- .../templates/udp_and_tcp_monitor_def.json | 347 ---- playbooks/hcp/toggle_ae_monitors.yml | 49 - playbooks/mininet/ae_tunnel.yml | 26 - playbooks/mininet/compile_p4.yml | 34 - playbooks/mininet/generate_inventory.yml | 50 - playbooks/mininet/generate_topology.yml | 32 - playbooks/mininet/local_inventory.yml | 23 - playbooks/mininet/set_ip-full.yml | 93 - playbooks/mininet/set_ip-lab_trial.yml | 50 - playbooks/mininet/set_ip-single-switch.yml | 41 - playbooks/mininet/setup-full.yml | 47 - playbooks/mininet/setup-lab_trial.yml | 38 - playbooks/mininet/setup-single_switch.yml | 36 - playbooks/mininet/setup_ae2mini_tunnel.yml | 14 - playbooks/mininet/setup_host.yml | 18 - playbooks/mininet/setup_mini2ae_tunnel.yml | 29 - playbooks/mininet/start_mininet.yml | 70 - .../mininet/templates/local_inventory.ini.j2 | 20 - .../templates/topology_template-full.yml.j2 | 284 --- .../topology_template-lab_trial.yml.j2 | 121 -- .../topology_template-single_switch.yml.j2 | 88 - playbooks/scenarios/full/all-tofino.yml | 58 - playbooks/scenarios/full/all.yml | 47 - playbooks/scenarios/full/discover_full.yml | 17 - playbooks/scenarios/full/packet-flood.yml | 128 -- playbooks/scenarios/gateway/all.yml | 86 - playbooks/scenarios/gateway/data-drop.yml | 48 - playbooks/scenarios/gateway/data-forward.yml | 35 - .../scenarios/gateway/data-inspection.yml | 45 - .../scenarios/general/drop_report_simple.yml | 2 +- .../siddhi/templates/convert_trpt.siddhi.j2 | 2 +- .../simple_ddos_clear_drop.siddhi.j2 | 2 +- .../templates/simple_ddos_detection.siddhi.j2 | 2 +- playbooks/tofino/setup_nodes-full.yml | 81 - playbooks/tofino/setup_orchestrator-full.yml | 16 - playbooks/tofino/setup_orchestrator.yml | 6 + snaps-hcp/OpenDistroElasticSearch/Readme.md | 4 - .../anomaly_detector_config.json | 80 - .../OpenDistroElasticSearch/data_mapping.json | 264 --- .../ddos_detection_and_mitigation_pipeline.md | 360 ---- .../packet_template-7.x.sh | 8 - .../udp_data_mapping_and_parsing_pipeline.md | 34 - .../udp_data_parsing.json | 1246 ------------ .../udp_data_parsing_pipeline.sh | 8 - snaps-hcp/Readme.md | 79 - snaps-hcp/setup/INSTALL.md | 1 - tests/trans_sec/analytics/__init__.py | 13 - tests/trans_sec/analytics/oinc_tests.py | 730 ------- .../controller/aggregate_controller_tests.py | 48 - .../controller/core_controller_tests.py | 47 - .../controller/ddos_sdn_controller_tests.py | 46 - .../controller/gateway_controller_tests.py | 48 - tests/trans_sec/mininet/__init__.py | 13 - tests/trans_sec/mininet/exercise_tests.py | 23 - tests/trans_sec/p4runtime_lib/__init__.py | 12 - .../p4runtime_lib/aggregate_switch_tests.py | 53 - .../p4runtime_lib/core_switch_tests.py | 54 - .../p4runtime_lib/gateway_switch_tests.py | 53 - trans_sec/analytics/__init__.py | 13 - trans_sec/analytics/oinc.py | 626 ------ trans_sec/controller/aggregate_controller.py | 25 +- trans_sec/controller/appcontroller.py | 127 -- trans_sec/controller/core_controller.py | 28 +- trans_sec/controller/ddos_sdn_controller.py | 62 - trans_sec/controller/gateway_controller.py | 143 -- trans_sec/controller/http_server_flask.py | 47 - trans_sec/device_software/hulk-attack.sh | 24 - trans_sec/mininet/__init__.py | 12 - trans_sec/mininet/apptopo.py | 89 - trans_sec/mininet/exercise.py | 343 ---- trans_sec/mininet/p4_mininet.py | 159 -- trans_sec/p4runtime_lib/__init__.py | 12 - trans_sec/p4runtime_lib/aggregate_switch.py | 148 -- trans_sec/p4runtime_lib/core_switch.py | 143 -- trans_sec/p4runtime_lib/gateway_switch.py | 349 ---- trans_sec/p4runtime_lib/helper.py | 297 --- trans_sec/p4runtime_lib/p4rt_switch.py | 624 ------ trans_sec/p4runtime_lib/p4runtime_switch.py | 143 -- 132 files changed, 65 insertions(+), 18591 deletions(-) delete mode 100644 automation/p4/mininet/aws-inst.tf delete mode 100644 automation/p4/mininet/aws-security.tf delete mode 100644 automation/p4/mininet/outputs.tf delete mode 100644 automation/p4/mininet/scenarios.tf delete mode 100644 automation/p4/mininet/setup.tf delete mode 100644 automation/p4/mininet/variables.tf delete mode 100755 bin/run_p4_mininet.py delete mode 100755 bin/start_ae.py delete mode 100644 docs/BUILD.md rename docs/{ => ae}/SIDDHI_AE_SETUP.md (100%) delete mode 100644 docs/mininet-example.tfvars delete mode 100755 p4/aggregate/aggregate.p4 delete mode 100755 p4/core/core.p4 delete mode 100755 p4/gateway/gateway.p4 delete mode 100644 playbooks/dependencies/bmv2.yml delete mode 100644 playbooks/dependencies/grpc.yml delete mode 100644 playbooks/dependencies/mininet.yml delete mode 100644 playbooks/dependencies/p4c.yml delete mode 100644 playbooks/dependencies/p4runtime.yml delete mode 100644 playbooks/dependencies/protobuf.yml delete mode 100644 playbooks/env-build/ae/env_build.yml delete mode 100644 playbooks/env-build/mininet/env_build.yml delete mode 100644 playbooks/general/templates/mininet_service.sh.j2 delete mode 100644 playbooks/hcp/ae_node_build.yml delete mode 100644 playbooks/hcp/conf_ae_elk.yml delete mode 100644 playbooks/hcp/delete_ae_history.yml delete mode 100644 playbooks/hcp/setup_ae_centos.yml delete mode 100644 playbooks/hcp/setup_ae_tcp_pipeline.yml delete mode 100644 playbooks/hcp/setup_ae_udp_and_tcp_pipeline.yml delete mode 100644 playbooks/hcp/setup_ae_udp_pipeline.yml delete mode 100644 playbooks/hcp/start_ae_elk.yml delete mode 100644 playbooks/hcp/templates/drop_report_hash_script.json delete mode 100644 playbooks/hcp/templates/index_pattern_request_body.json delete mode 100644 playbooks/hcp/templates/opendistro_elasticsearch.sh delete mode 100644 playbooks/hcp/templates/opendistro_kibana.sh delete mode 100644 playbooks/hcp/templates/query_monitor_id.json delete mode 100644 playbooks/hcp/templates/sdn_attack_webhook.json delete mode 100644 playbooks/hcp/templates/tcp/tcp_data_mapping.json delete mode 100644 playbooks/hcp/templates/tcp/tcp_data_parsing.json delete mode 100644 playbooks/hcp/templates/tcp/tcp_monitor_def.json delete mode 100644 playbooks/hcp/templates/udp/udp_data_mapping.json delete mode 100644 playbooks/hcp/templates/udp/udp_data_parsing.json delete mode 100644 playbooks/hcp/templates/udp/udp_monitor_def.json delete mode 100644 playbooks/hcp/templates/udp_and_tcp_data_mapping.json delete mode 100644 playbooks/hcp/templates/udp_and_tcp_data_parsing.json delete mode 100644 playbooks/hcp/templates/udp_and_tcp_monitor_def.json delete mode 100644 playbooks/hcp/toggle_ae_monitors.yml delete mode 100644 playbooks/mininet/ae_tunnel.yml delete mode 100644 playbooks/mininet/compile_p4.yml delete mode 100644 playbooks/mininet/generate_inventory.yml delete mode 100644 playbooks/mininet/generate_topology.yml delete mode 100644 playbooks/mininet/local_inventory.yml delete mode 100644 playbooks/mininet/set_ip-full.yml delete mode 100644 playbooks/mininet/set_ip-lab_trial.yml delete mode 100644 playbooks/mininet/set_ip-single-switch.yml delete mode 100644 playbooks/mininet/setup-full.yml delete mode 100644 playbooks/mininet/setup-lab_trial.yml delete mode 100644 playbooks/mininet/setup-single_switch.yml delete mode 100644 playbooks/mininet/setup_ae2mini_tunnel.yml delete mode 100644 playbooks/mininet/setup_host.yml delete mode 100644 playbooks/mininet/setup_mini2ae_tunnel.yml delete mode 100644 playbooks/mininet/start_mininet.yml delete mode 100644 playbooks/mininet/templates/local_inventory.ini.j2 delete mode 100755 playbooks/mininet/templates/topology_template-full.yml.j2 delete mode 100644 playbooks/mininet/templates/topology_template-lab_trial.yml.j2 delete mode 100644 playbooks/mininet/templates/topology_template-single_switch.yml.j2 delete mode 100644 playbooks/scenarios/full/all-tofino.yml delete mode 100644 playbooks/scenarios/full/all.yml delete mode 100644 playbooks/scenarios/full/discover_full.yml delete mode 100644 playbooks/scenarios/full/packet-flood.yml delete mode 100644 playbooks/scenarios/gateway/all.yml delete mode 100644 playbooks/scenarios/gateway/data-drop.yml delete mode 100644 playbooks/scenarios/gateway/data-forward.yml delete mode 100644 playbooks/scenarios/gateway/data-inspection.yml delete mode 100644 playbooks/tofino/setup_nodes-full.yml delete mode 100644 playbooks/tofino/setup_orchestrator-full.yml delete mode 100644 snaps-hcp/OpenDistroElasticSearch/Readme.md delete mode 100644 snaps-hcp/OpenDistroElasticSearch/anomaly_detector_config.json delete mode 100644 snaps-hcp/OpenDistroElasticSearch/data_mapping.json delete mode 100644 snaps-hcp/OpenDistroElasticSearch/ddos_detection_and_mitigation_pipeline.md delete mode 100755 snaps-hcp/OpenDistroElasticSearch/packet_template-7.x.sh delete mode 100644 snaps-hcp/OpenDistroElasticSearch/udp_data_mapping_and_parsing_pipeline.md delete mode 100644 snaps-hcp/OpenDistroElasticSearch/udp_data_parsing.json delete mode 100755 snaps-hcp/OpenDistroElasticSearch/udp_data_parsing_pipeline.sh delete mode 100644 snaps-hcp/Readme.md delete mode 100644 snaps-hcp/setup/INSTALL.md delete mode 100644 tests/trans_sec/analytics/__init__.py delete mode 100644 tests/trans_sec/analytics/oinc_tests.py delete mode 100644 tests/trans_sec/controller/aggregate_controller_tests.py delete mode 100644 tests/trans_sec/controller/core_controller_tests.py delete mode 100644 tests/trans_sec/controller/ddos_sdn_controller_tests.py delete mode 100644 tests/trans_sec/controller/gateway_controller_tests.py delete mode 100644 tests/trans_sec/mininet/__init__.py delete mode 100644 tests/trans_sec/mininet/exercise_tests.py delete mode 100644 tests/trans_sec/p4runtime_lib/__init__.py delete mode 100644 tests/trans_sec/p4runtime_lib/aggregate_switch_tests.py delete mode 100644 tests/trans_sec/p4runtime_lib/core_switch_tests.py delete mode 100644 tests/trans_sec/p4runtime_lib/gateway_switch_tests.py delete mode 100644 trans_sec/analytics/__init__.py delete mode 100644 trans_sec/analytics/oinc.py delete mode 100644 trans_sec/controller/appcontroller.py delete mode 100755 trans_sec/controller/gateway_controller.py delete mode 100755 trans_sec/device_software/hulk-attack.sh delete mode 100644 trans_sec/mininet/__init__.py delete mode 100644 trans_sec/mininet/apptopo.py delete mode 100644 trans_sec/mininet/exercise.py delete mode 100644 trans_sec/mininet/p4_mininet.py delete mode 100644 trans_sec/p4runtime_lib/__init__.py delete mode 100644 trans_sec/p4runtime_lib/aggregate_switch.py delete mode 100644 trans_sec/p4runtime_lib/core_switch.py delete mode 100644 trans_sec/p4runtime_lib/gateway_switch.py delete mode 100644 trans_sec/p4runtime_lib/helper.py delete mode 100644 trans_sec/p4runtime_lib/p4rt_switch.py delete mode 100644 trans_sec/p4runtime_lib/p4runtime_switch.py diff --git a/README.md b/README.md index d4343754..abfeb047 100644 --- a/README.md +++ b/README.md @@ -41,4 +41,4 @@ Questions? Just send us an email at - playbooks - The Ansible Playbooks used by automation - snaps-hcp - Documentation on snaps-hcp which is no longer being used - tests - the Python unit test directory -- trans_sec - the project's top-level Python package \ No newline at end of file +- trans_sec - the project's top-level Python package diff --git a/automation/env-build/aws-inst.tf b/automation/env-build/aws-inst.tf index a7422952..73f6d9d8 100644 --- a/automation/env-build/aws-inst.tf +++ b/automation/env-build/aws-inst.tf @@ -75,11 +75,6 @@ ${var.ANSIBLE_PB_PATH}/env-build/${var.env_type}/env_build.yml \ --extra-vars "\ aws_access_key=${var.access_key} aws_secret_key=${var.secret_key} -grpc_version=${var.grpc_version} -p4c_version=${var.p4c_version} -protobuf_version=${var.protobuf_version} -pi_version=${var.pi_version} -bm_version=${var.bm_version} bf_sde_version=${var.bf_sde_version} bf_sde_profile=${var.bf_sde_profile} bf_sde_s3_bucket=${var.bf_sde_s3_bucket} diff --git a/automation/env-build/variables.tf b/automation/env-build/variables.tf index 536cad00..a04a1d32 100644 --- a/automation/env-build/variables.tf +++ b/automation/env-build/variables.tf @@ -16,14 +16,14 @@ variable "access_key" {} variable "secret_key" {} variable "build_id" {} variable "ec2_region" {} -variable "env_type" {default = "mininet"} +variable "env_type" {default = "tofino"} # Dependency version only for tofino environments variable "bf_sde_s3_bucket" {default = "null"} # Optional Variables variable "bf_sde_version" {default = "9.2.0"} -variable "bf_sde_profile" {default = "p4_runtime_profile"} +variable "bf_sde_profile" {default = "p416_examples_profile"} variable "create_ami" {default = "yes"} variable "public_key_file" {default = "~/.ssh/id_rsa.pub"} variable "private_key_file" {default = "~/.ssh/id_rsa"} @@ -45,12 +45,6 @@ variable "base_20_ami" {default = "ami-03d5c68bab01f3496"} variable "instance_type" {default = "t2.2xlarge"} variable "run_tests" {default = "yes"} -# Dependency versions only for mininet environments -variable "grpc_version" {default = "v1.19.1"} -variable "p4c_version" {default = "fbe395bbf1eed9653323ac73b20cf6c06af2121e"} -variable "protobuf_version" {default = "3.6.x"} -variable "pi_version" {default = "1539ecd8a50c159b011d9c5a9c0eba99f122a845"} -variable "bm_version" {default = "16c699953ee02306731ebf9a9241ea9fe3bbdc8c"} variable "remote_scripts_dir" {default = "/etc/transparent-security"} # Playbook Constants diff --git a/automation/p4/mininet/aws-inst.tf b/automation/p4/mininet/aws-inst.tf deleted file mode 100644 index 77412b6f..00000000 --- a/automation/p4/mininet/aws-inst.tf +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# AWS EC2 Instance -resource "aws_instance" "transparent-security-mininet-integration" { - ami = var.mininet_ami - instance_type = var.instance_type - key_name = aws_key_pair.transparent-security-pk.key_name - - tags = { - Name = "transparent-security-transparent-security-${var.scenario_name}-${var.build_id}" - } - - security_groups = [aws_security_group.transparent-security-img-sg.name] - associate_public_ip_address = true - - # Used to ensure host is really up before attempting to apply ansible playbooks - provisioner "remote-exec" { - inline = [ - "sudo echo 'transparent-security mininet integration CI' > ~/motd", - ] - } - - # Remote connection info for remote-exec - connection { - host = self.public_ip - type = "ssh" - user = var.sudo_user - private_key = file(var.private_key_file) - } - - root_block_device { - volume_size = "50" - } -} - -resource "aws_instance" "transparent-security-hcp-instance" { - count = var.scenario_name == "lab_trial" ? 1 : 0 - ami = var.hcp_ami - instance_type = var.instance_type - key_name = aws_key_pair.transparent-security-pk.key_name - - tags = { - Name = "transparent-security-hcp-${var.scenario_name}-${var.build_id}" - } - - security_groups = [aws_security_group.transparent-security-img-sg.name, aws_security_group.transparent-security-hcp-img-sg.name ] - associate_public_ip_address = true - - # Used to ensure host is really up before attempting to apply ansible playbooks - provisioner "remote-exec" { - inline = [ - "sudo echo 'transparent-security-hcp integration' > ~/motd", - ] - } - - # Remote connection info for remote-exec - connection { - host = self.public_ip - type = "ssh" - user = var.hcp_sudo_user - private_key = file(var.private_key_file) - } -} diff --git a/automation/p4/mininet/aws-security.tf b/automation/p4/mininet/aws-security.tf deleted file mode 100644 index c3d82009..00000000 --- a/automation/p4/mininet/aws-security.tf +++ /dev/null @@ -1,89 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# AWS Credentials -provider "aws" { - access_key = var.access_key - secret_key = var.secret_key - region = var.ec2_region -} - -# Note: Script will fail if another process is leveraging the same build_id -resource "aws_security_group" "transparent-security-img-sg" { - name = "transparent-security-${var.scenario_name}-${var.build_id}" - ingress { - cidr_blocks = ["0.0.0.0/0"] - from_port = 22 - to_port = 22 - protocol = "tcp" - } - - ingress { - cidr_blocks = ["0.0.0.0/0"] - from_port = var.sdn_port - to_port = var.sdn_port - protocol = "tcp" - } - - ingress { - cidr_blocks = ["0.0.0.0/0"] - from_port = 50051 - to_port = 50056 - protocol = "tcp" - } - - // Terraform removes the default rule - egress { - from_port = 0 - to_port = 0 - protocol = "-1" - cidr_blocks = ["0.0.0.0/0"] - } -} - -resource "aws_security_group" "transparent-security-hcp-img-sg" { - name = "transparent-security-hcp-${var.scenario_name}-${var.build_id}" - ingress { - cidr_blocks = ["0.0.0.0/0"] - from_port = 22 - to_port = 22 - protocol = "tcp" - } - - ingress { - cidr_blocks = ["0.0.0.0/0"] - from_port = var.sdn_port - to_port = var.sdn_port - protocol = "tcp" - } - - ingress { - cidr_blocks = ["0.0.0.0/0"] - from_port = 50051 - to_port = 50056 - protocol = "tcp" - } - - // Terraform removes the default rule - egress { - from_port = 0 - to_port = 0 - protocol = "-1" - cidr_blocks = ["0.0.0.0/0"] - } -} - -# AWS EC2 Instance Public Key -resource "aws_key_pair" "transparent-security-pk" { - public_key = file(var.public_key_file) -} diff --git a/automation/p4/mininet/outputs.tf b/automation/p4/mininet/outputs.tf deleted file mode 100644 index acf5757f..00000000 --- a/automation/p4/mininet/outputs.tf +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Outputs -output "ip" { - value = aws_instance.transparent-security-mininet-integration.public_ip -} - -output "hcp-ip" { - value = var.scenario_name == "lab_trial" ? aws_instance.transparent-security-hcp-instance.0.public_ip : "n/a" -} - -output "hcp-private-ip" { - value = var.scenario_name == "lab_trial" ? aws_instance.transparent-security-hcp-instance.0.private_ip: "n/a" -} diff --git a/automation/p4/mininet/scenarios.tf b/automation/p4/mininet/scenarios.tf deleted file mode 100644 index d45c7f62..00000000 --- a/automation/p4/mininet/scenarios.tf +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -resource "null_resource" "transparent-security-run-scenario-tests" { - depends_on = [ - null_resource.transparent-security-start-sim, - null_resource.transparent-security-hcp-tunnel, - ] - - provisioner "remote-exec" { - inline = [ - "sudo pip install ansible", - "${var.ANSIBLE_CMD} -i ${var.remote_inventory_file} ${var.remote_scenario_pb_dir}/${var.scenario_name}/${var.test_case}.yml", - ] - } - - connection { - host = aws_instance.transparent-security-mininet-integration.public_ip - type = "ssh" - user = var.sudo_user - private_key = file(var.private_key_file) - } -} diff --git a/automation/p4/mininet/setup.tf b/automation/p4/mininet/setup.tf deleted file mode 100644 index 27e1b48b..00000000 --- a/automation/p4/mininet/setup.tf +++ /dev/null @@ -1,150 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Create and inject own SSH keys for being able to SSH into self or mininet hosts -resource "null_resource" "transparent-security-host-ssh-setup" { - provisioner "remote-exec" { - inline = [ - "ssh-keygen -t rsa -N '' -f ~/.ssh/id_rsa", - "touch ~/.ssh/authorized_keys", - "chmod 600 ~/.ssh/authorized_keys", - "cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys", - ] - } - connection { - host = aws_instance.transparent-security-mininet-integration.public_ip - type = "ssh" - user = var.sudo_user - private_key = file(var.private_key_file) - } -} - -locals { - local_ansible_inventory_file = "~/tps-mininet-setup-${var.scenario_name}-${var.build_id}.ini" -} - -#Create a local inventory to store variables and public IP of remote machine -resource "null_resource" "transparent-security-local-inventory" { - depends_on = [null_resource.transparent-security-host-ssh-setup] - provisioner "local-exec" { - command = <-.yml -variable "test_case" {default = "all"} diff --git a/automation/p4/tofino/aws-insts.tf b/automation/p4/tofino/aws-insts.tf index 4e39b7b2..448d7ea3 100644 --- a/automation/p4/tofino/aws-insts.tf +++ b/automation/p4/tofino/aws-insts.tf @@ -13,6 +13,10 @@ locals { ami = var.p4_arch == "tna" ? var.tofino.bfrt_ami : var.tofino.p4rt_ami + dflt_start_script = ["sudo echo 'tps-orchestrator/controller' > ~/motd"] + // Placeholder for custom startup options for BFRT images + tofino_img_start_script = local.dflt_start_script + start_script = var.p4_arch == "tna" ? local.tofino_img_start_script : local.dflt_start_script } # Orchestrator/SDN Controller Instance @@ -29,9 +33,7 @@ resource "aws_instance" "orchestrator" { associate_public_ip_address = true provisioner "remote-exec" { - inline = [ - "sudo echo 'tps-orchestrator/controller' > ~/motd", - ] + inline = local.start_script } # Remote connection info for remote-exec @@ -45,8 +47,8 @@ resource "aws_instance" "orchestrator" { # Determine switch and node count locals { - switch_count = var.scenario_name == "full" ? var.num_switches_full : var.scenario_name == "lab_trial" ? var.num_switches_lab_trial : var.num_switches_single - node_count = var.scenario_name == "full" ? var.num_nodes_full : var.scenario_name == "lab_trial" ? var.num_nodes_lab_trial : var.num_nodes_single + switch_count = var.scenario_name == "lab_trial" ? var.num_switches_lab_trial : var.num_switches_single + node_count = var.scenario_name == "lab_trial" ? var.num_nodes_lab_trial : var.num_nodes_single } # Tofino Model Switch Instances @@ -63,6 +65,18 @@ resource "aws_instance" "tps-switch" { security_groups = [aws_security_group.tps.name] associate_public_ip_address = true + + provisioner "remote-exec" { + inline = local.start_script + } + + # Remote connection info for remote-exec + connection { + host = self.public_ip + type = "ssh" + user = var.switch_user + private_key = file(var.private_key_file) + } } # Third octet of the subnet IPv4 value diff --git a/automation/p4/tofino/setup.tf b/automation/p4/tofino/setup.tf index 165d6515..16d430ec 100644 --- a/automation/p4/tofino/setup.tf +++ b/automation/p4/tofino/setup.tf @@ -14,46 +14,32 @@ locals { sdn_ip = aws_instance.orchestrator.private_ip - # For full & lab_trial scenarios - core_switch_ip = var.scenario_name == "full" || var.scenario_name == "lab_trial" ? aws_instance.tps-switch.0.private_ip: "n/a" - core_tun1_ip = var.scenario_name == "full" || var.scenario_name == "lab_trial" ? aws_network_interface.switch_tun_1.0.private_ip: "n/a" - core_tun1_mac = var.scenario_name == "full" || var.scenario_name == "lab_trial" ? aws_network_interface.switch_tun_1.0.mac_address: "n/a" - agg_switch_ip = var.scenario_name == "full" || var.scenario_name == "lab_trial" ? aws_instance.tps-switch.1.private_ip: "n/a" - agg_tun1_ip = var.scenario_name == "full" || var.scenario_name == "lab_trial" ? aws_network_interface.switch_tun_1.1.private_ip: "n/a" - agg_tun1_mac = var.scenario_name == "full" || var.scenario_name == "lab_trial" ? aws_network_interface.switch_tun_1.1.mac_address: "n/a" - - # For full scenario - gateway_1_ip = var.scenario_name == "full" ? aws_instance.tps-switch.2.private_ip: "n/a" - gateway_2_ip = var.scenario_name == "full" ? aws_instance.tps-switch.3.private_ip: "n/a" - gateway_3_ip = var.scenario_name == "full" ? aws_instance.tps-switch.4.private_ip: "n/a" - camera_1_ip = var.scenario_name == "full" ? aws_instance.node.0.private_ip: "n/a" - nas_1_ip = var.scenario_name == "full" ? aws_instance.node.1.private_ip: "n/a" - game_1_ip = var.scenario_name == "full" ? aws_instance.node.2.private_ip: "n/a" - camera_2_ip = var.scenario_name == "full" ? aws_instance.node.3.private_ip: "n/a" - game_2_ip = var.scenario_name == "full" ? aws_instance.node.4.private_ip: "n/a" - camera_3_ip = var.scenario_name == "full" ? aws_instance.node.5.private_ip: "n/a" - game_3_ip = var.scenario_name == "full" ? aws_instance.node.6.private_ip: "n/a" - # For lab_trial scenario + core_switch_ip = var.scenario_name == "lab_trial" ? aws_instance.tps-switch.0.private_ip: "n/a" + core_tun1_ip = var.scenario_name == "lab_trial" ? aws_network_interface.switch_tun_1.0.private_ip: "n/a" + core_tun1_mac = var.scenario_name == "lab_trial" ? aws_network_interface.switch_tun_1.0.mac_address: "n/a" + agg_switch_ip = var.scenario_name == "lab_trial" ? aws_instance.tps-switch.1.private_ip: "n/a" + agg_tun1_ip = var.scenario_name == "lab_trial" ? aws_network_interface.switch_tun_1.1.private_ip: "n/a" + agg_tun1_mac = var.scenario_name == "lab_trial" ? aws_network_interface.switch_tun_1.1.mac_address: "n/a" ae_ip = var.scenario_name == "lab_trial" ? aws_instance.ae.private_ip: "n/a" ae_tun1_ip = var.scenario_name == "lab_trial" ? aws_network_interface.ae_tun_1.private_ip: "n/a" ae_tun1_mac = var.scenario_name == "lab_trial" ? aws_network_interface.ae_tun_1.mac_address: "n/a" # For single-switch scenario - switch_ip = var.scenario_name == "full" ? "n/a" : aws_instance.tps-switch.0.private_ip - switch_tun1_ip = var.scenario_name == "full" ? "n/a" : aws_network_interface.switch_tun_1.0.private_ip - switch_tun1_mac = var.scenario_name == "full" ? "n/a" : aws_network_interface.switch_tun_1.0.mac_address - clone_ip = var.scenario_name == "full" ? "n/a" : aws_instance.node.2.private_ip - clone_tun1_ip = var.scenario_name == "full" ? "n/a" : aws_network_interface.node_tun_1.2.private_ip - clone_tun1_mac = var.scenario_name == "full" ? "n/a" : aws_network_interface.node_tun_1.2.mac_address + switch_ip = aws_instance.tps-switch.0.private_ip + switch_tun1_ip = aws_network_interface.switch_tun_1.0.private_ip + switch_tun1_mac = aws_network_interface.switch_tun_1.0.mac_address + clone_ip = aws_instance.node.2.private_ip + clone_tun1_ip = aws_network_interface.node_tun_1.2.private_ip + clone_tun1_mac = aws_network_interface.node_tun_1.2.mac_address # For single_switch & lab_trial scenarios - host1_ip = var.scenario_name == "full" ? "n/a" : aws_instance.node.0.private_ip - host1_tun1_ip = var.scenario_name == "full" ? "n/a" : aws_network_interface.node_tun_1.0.private_ip - host1_tun1_mac = var.scenario_name == "full" ? "n/a" : aws_network_interface.node_tun_1.0.mac_address - host2_ip = var.scenario_name == "full" ? "n/a" : aws_instance.node.1.private_ip - host2_tun1_ip = var.scenario_name == "full" ? "n/a" : aws_network_interface.node_tun_1.1.private_ip - host2_tun1_mac = var.scenario_name == "full" ? "n/a" : aws_network_interface.node_tun_1.1.mac_address + host1_ip = aws_instance.node.0.private_ip + host1_tun1_ip = aws_network_interface.node_tun_1.0.private_ip + host1_tun1_mac = aws_network_interface.node_tun_1.0.mac_address + host2_ip = aws_instance.node.1.private_ip + host2_tun1_ip = aws_network_interface.node_tun_1.1.private_ip + host2_tun1_mac = aws_network_interface.node_tun_1.1.mac_address # For lab_trial scenarios lab_inet_ip = var.scenario_name == "lab_trial" ? aws_instance.node.2.private_ip: "n/a" @@ -92,7 +78,7 @@ EOT } resource "null_resource" "tps-sim-setup-orch-single-switch" { - count = var.scenario_name == "full" || var.scenario_name == "lab_trial" ? 0 : 1 + count = var.scenario_name == "lab_trial" ? 0 : 1 depends_on = [ aws_instance.tps-switch, aws_instance.node, @@ -192,7 +178,7 @@ EOT } locals { - setup_pb = var.scenario_name == "full" || var.scenario_name == "lab_trial" ? "setup_nodes-${var.scenario_name}.yml" : "setup_nodes-single_switch.yml" + setup_pb = var.scenario_name == "lab_trial" ? "setup_nodes-${var.scenario_name}.yml" : "setup_nodes-single_switch.yml" } resource "null_resource" "tps-tofino-setup-nodes" { diff --git a/automation/p4/tofino/variables.tf b/automation/p4/tofino/variables.tf index c82b9fa1..6d1ace96 100644 --- a/automation/p4/tofino/variables.tf +++ b/automation/p4/tofino/variables.tf @@ -36,25 +36,19 @@ variable "tofino" { } } -variable "siddhi_ae_ami" {default = "ami-08ec1da30d3413105"} -//variable "siddhi_ae_ami" {default = "ami-0e7d14e85b77459c4"} -//variable "siddhi_ae_ami" {default = "ami-06ea3285e31e16516"} -//variable "siddhi_ae_ami" {default = "ami-00876f36fe8f97733"} +variable "siddhi_ae_ami" {default = "ami-05bb35a9f06141f6b"} variable "switch_instance_type" {default = "t2.2xlarge"} variable "ae_instance_type" {default = "t2.2xlarge"} variable "orch_instance_type" {default = "t2.2xlarge"} variable "node_instance_type" {default = "t2.small"} -variable "num_switches_full" {default = 5} variable "num_switches_single" {default = 1} variable "num_switches_lab_trial" {default = 2} -variable "num_nodes_full" {default = 9} variable "num_nodes_single" {default = 3} variable "num_nodes_lab_trial" {default = 3} # Variables for ansible playbooks variable "ANSIBLE_CMD" {default = "export ANSIBLE_HOST_KEY_CHECKING=False; ansible-playbook"} -variable "SETUP_ORCH_FULL" {default = "../../../playbooks/tofino/setup_orchestrator-full.yml"} variable "SETUP_ORCH_SINGLE_SWITCH" {default = "../../../playbooks/tofino/setup_orchestrator-single_switch.yml"} variable "SETUP_ORCH_LAB_TRIAL" {default = "../../../playbooks/tofino/setup_orchestrator-lab_trial.yml"} variable "START_SERVICE" {default = "../../../playbooks/general/start_service.yml"} @@ -82,5 +76,5 @@ variable "p4_arch" {default = "tna"} variable "from_hw" {default = "False"} variable "setup_nodes_pb" {default = "setup_nodes.yml"} -variable "scenario_name" {default = "full"} +variable "scenario_name" {default = "lab_trial"} variable "test_case" {default = "all"} diff --git a/bin/run_p4_mininet.py b/bin/run_p4_mininet.py deleted file mode 100755 index e452a002..00000000 --- a/bin/run_p4_mininet.py +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import argparse -import json -import logging -import sys -import yaml - -from trans_sec.mininet.exercise import ExerciseRunner - -logger = logging.getLogger('') - - -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument('-t', '--topo', help='Path to topology json', - type=str, required=True) - parser.add_argument('-l', '--log-dir', type=str, required=True, - default=None) - parser.add_argument('-lf', '--log-file', type=str, required=False, - default='run_p4_mininet.log') - parser.add_argument('-p', '--pcap-dir', type=str, required=False, - default=None) - parser.add_argument('-j', '--switch_json', type=str, required=False) - parser.add_argument('-c', '--start-cli', type=bool, required=False, - default=None) - parser.add_argument('-d', '--daemon', help='Run device daemon on hosts.', - type=bool, required=False, default=False) - return parser.parse_args() - - -def read_yaml_file(config_file_path): - """ - Reads a yaml file and returns a dict representation of it - :return: a dict of the yaml file - """ - logger.debug('Attempting to load configuration file - ' + config_file_path) - config_file = None - try: - with open(config_file_path, 'r') as config_file: - config = yaml.safe_load(config_file) - logger.info('Loaded configuration') - return config - finally: - if config_file: - logger.info('Closing configuration file') - config_file.close() - - -if __name__ == '__main__': - args = get_args() - - if args.log_file: - log_file = '{}/{}'.format(args.log_dir, args.log_file) - logging.basicConfig(level=logging.DEBUG, filename=log_file) - else: - logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) - - topo_file = args.topo - if topo_file.endswith('json'): - with open(topo_file, 'r') as f: - topo = json.load(f) - else: - topo = read_yaml_file(topo_file) - - exercise = ExerciseRunner( - topo, args.log_dir, args.pcap_dir, args.switch_json, args.start_cli) - exercise.run_exercise() - - logger.info('Exercise Runner running indefinitely') - while True: - pass diff --git a/bin/sdn_controller.py b/bin/sdn_controller.py index 8a3171bd..027976a9 100755 --- a/bin/sdn_controller.py +++ b/bin/sdn_controller.py @@ -22,8 +22,7 @@ from trans_sec.controller.aggregate_controller import AggregateController from trans_sec.controller.core_controller import CoreController from trans_sec.controller.ddos_sdn_controller import ( - DdosSdnController, GATEWAY_CTRL_KEY, AGG_CTRL_KEY, CORE_CTRL_KEY) -from trans_sec.controller.gateway_controller import GatewayController + DdosSdnController, AGG_CTRL_KEY, CORE_CTRL_KEY) FORMAT = '%(levelname)s %(asctime)-15s %(filename)s %(message)s' logger = logging.getLogger('sdn_controller') @@ -50,8 +49,7 @@ def get_args(): parser.add_argument('-ld', '--log-dir', type=str, required=True, default=default_logs) parser.add_argument('-t', '--topo', help='Path to topology json', - required=False, - default='../mininet-start/conf/topology_proposed.json') + required=True) parser.add_argument('-p', '--platform', help='Switch Platform', required=True, type=str, choices=['bmv2', 'tofino']) parser.add_argument('-s', '--switch-config-dir', dest='switch_config_dir', @@ -78,8 +76,7 @@ def get_args(): help='When set, the controller will not attempt to ' 'load the P4 program onto the switches') parser.add_argument('-ct', '--controller-type', type=str, required=True, - choices=['gateway', 'aggregate', 'core', 'full', - 'lab_trial'], + choices=['aggregate', 'core', 'full', 'lab_trial'], help='When not set, all controllers will attempt to ' 'start') return parser.parse_args() @@ -122,15 +119,6 @@ def main(): 'Starting SDN Controller with topology - [%s] and load_p4 flag - [%s]', topo, eval(args.load_p4)) controllers = dict() - if (args.controller_type == 'gateway' - or args.controller_type == 'full'): - logger.info('Instantiating a GatewayController') - controllers[GATEWAY_CTRL_KEY] = GatewayController( - platform=args.platform, - p4_build_out=args.switch_config_dir, - topo=topo, - log_dir=args.log_dir, - load_p4=eval(args.load_p4)) if (args.controller_type == 'aggregate' or args.controller_type == 'lab_trial' or args.controller_type == 'full'): diff --git a/bin/start_ae.py b/bin/start_ae.py deleted file mode 100755 index 8d50109f..00000000 --- a/bin/start_ae.py +++ /dev/null @@ -1,117 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import argparse -import logging -import sys - -import pydevd - -from trans_sec.analytics.oinc import Oinc, SimpleAE, LoggerAE, IntLoggerAE -from trans_sec.utils.http_session import HttpSession - -logger = logging.getLogger('start_ae') -FORMAT = '%(levelname)s %(asctime)-15s %(filename)s %(lineno)d %(message)s' - - -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument( - '-l', '--loglevel', - help='Log Level defaults to INFO', - required=False, default='INFO') - parser.add_argument('-f', '--logfile', - help='File to log to defaults to console', - required=False, default=None) - parser.add_argument('-i', '--interface', - help='Linux interface to listen on', required=True) - parser.add_argument('-di', '--drop-interface', dest='drop_interface', - help='Linux interface to listen for Drop Reports', - required=False) - parser.add_argument('-dh', '--debug-host', dest='debug_host', - help='remote debugging host IP') - parser.add_argument('-dp', '--debug-port', dest='debug_port', default=5678, - help='the remote debugging port') - parser.add_argument('-s', '--sdn-url', dest='sdn_url', required=True, - help='the URL to the SDN controller') - parser.add_argument('-t', '--type', dest='type', required=True, - choices=['OINC', 'SIMPLE', 'LOGGING', 'INT'], - help='Acceptable values OINC|SIMPLE|LOGGING|INT') - parser.add_argument('-c', '--sdn-attack-ctx', dest='sdn_ctx', - required=False, default='gwAttack', - help='the URL to the SDN controller') - parser.add_argument('-pc', '--packet-count', required=False, type=int, - default=100, - help='Number of packets to hit alert within the ' - 'sample-interval') - parser.add_argument('-si', '--sample-interval', required=False, type=int, - default=60, - help='Number of seconds the packet count needs to hit ' - 'to trigger alert') - parser.add_argument('-pb', '--packet-bytes', required=False, type=int, - default=50000, - help='Number of packet bytes to hit alert within the ' - 'sample-interval') - return parser.parse_args() - - -def main(): - args = get_args() - - # Initialize logger - numeric_level = getattr(logging, args.loglevel.upper(), None) - - if args.logfile: - logging.basicConfig(format=FORMAT, level=numeric_level, - filename=args.logfile) - else: - logging.basicConfig(format=FORMAT, level=numeric_level) - - # Setup remote debugging - if args.debug_host: - pydevd.settrace(host=args.debug_host, port=int(args.debug_port), - stdoutToServer=True, stderrToServer=True, - suspend=False) - logger.info('Starting Oinc with SDN Controller url [%s]', args.sdn_url) - - ae = None - - logger.info('Retrieving http session from url - [%s]', args.sdn_url) - http_session = HttpSession(args.sdn_url) - - if args.type == 'SIMPLE': - logger.info('SimpleAE instantiated') - logger.debug('SDN Context - [%s]', args.sdn_ctx) - ae = SimpleAE(http_session, packet_count=args.packet_count, - sample_interval=args.sample_interval, - sdn_attack_context=args.sdn_ctx, - byte_count=args.packet_bytes) - elif args.type == 'OINC': - logger.info('Oinc instantiated') - ae = Oinc(http_session) - elif args.type == 'LOGGING': - logger.info('LoggerAE instantiated') - ae = LoggerAE(http_session) - elif args.type == 'INT': - logger.info('LoggerAE instantiated') - ae = IntLoggerAE(http_session) - - logger.info('Begin sniffing for INT on [%s] and Drop Reports on [%s]', - args.interface, args.drop_interface) - ae.start_sniffing(args.interface, args.drop_interface) - sys.stdout.flush() - - -if __name__ == '__main__': - main() diff --git a/docs/BUILD.md b/docs/BUILD.md deleted file mode 100644 index 269b5008..00000000 --- a/docs/BUILD.md +++ /dev/null @@ -1,465 +0,0 @@ -# transparent-security automation - -The scripts outlined here have been designed to be executed within a CI server - -## Table of Contents - -1. Introduction -2. Client system setup -3. [Optional] Create an OS instance for running the mininet simulator -4. Run mininet simulator (using Terraform) -5. [Optional] Run mininet simulator (using ansible) -6. [Optional] Run mininet simulator on a local VM -7. Using Mininet - -## 1. Introduction - -This document provides instructuions to: - -1. Building a transparent-security environment on AWS -1. Setting up Mininet and AWS - -## 2. Client system setup - -When running builds and simulations this project recommends running them on a cloud infrastructure. These instructions are using AWS EC2. With minor changes these could be run on other cloud types. - -You will use a local system for: - -* Running Ansible and Terraform to orchestrate the simulator -* Downloading the Transparent Security source -* Configuring the input file - -The local system can be Linux, Mac OS or Windows. We provide examples for Linux. This has also been testing on Mac OS. - -### 2.1 Install dependencies on local client - -Install git, python-ansible and terraform. - -#### 2.1.1 Install git - -Install the git client. - -On Ubuntu: - -```bash -sudo apt update -sudo apt install git -``` - -#### 2.1.2 Install Python Ansible - -* Python 3.6+ is installed -* The python3-pip package has been installed -* The Python ansible >=2.7.5 package has been installed - -Use python3-pip to install ansible >=2.7.5. - -On Ubuntu run: - -```bash -sudo apt update -sudo apt install python3-pip -sudo pip install ansible -``` - -Validate the ansible version: - -```bash -ansible --version -``` - -Example output: - -```bash -ansible 2.9.2 -. -. -. - python version = 3.6.8 (default, Nov 9 2019, 05:55:08) [GCC 4.2.1 Compatible Apple LLVM 11.0.0 (clang-1100.0.32.4) (-macos10.15-objc-s -``` - -#### 2.1.2 Install Terraform - -See terraform documentation for installation instructions. - -[Terraform Download page](https://www.terraform.io/downloads.html) - -### 2.2. Download Transparent Secuirty from Git - -Download the latest source from [Transparent Security GitHub](https://github.com/cablelabs/transparent-security) - -```bash -git clone https://github.com/cablelabs/transparent-security -``` - -Example output: - -```bash -Cloning into 'transparent-security'... -. -. -. -Resolving deltas: 100% (554/554), done. -``` - -### 2.3. Obtain credentials to AWS - -To use the directions, you will need an account on [AWS](https://aws.amazon.com/) with API access keys. - -### 2.4 Customize the variable file for your environment - -Copy the example variable file docs/mininet-example.tfvars to a working directory and make changes to adapt the file to your local environment. - -| Variable | Description | Type | Example | -|------------------|-------------------------------------------------------------------------------------------------------------------------------------------|--------|---------------------------------------------------------| -| build_id | This value must be unique to ensure multiple jobs can be run simultaneously from multiple hosts | string | build_id = "test-1" | -| access_key | Amazon EC2 access key | string | access_key = "AKIAIOSFODNN7EXAMPLE" | -| secret_key | Amazon EC2 secret key | string | secret_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" | -| ec2_region | Amazon EC2 region | string | ec2_region = "us-west-2" | -| public_key_file | Used to inject into the VM for SSH access with the user'ubuntu' (defaults to ~/.ssh/id_rsa.pub) | string | public_key_file = "~/.ssh/id_rsa.pub" | -| private_key_file | Used to access the VM via SSH with the user 'ubuntu' (defaults to ~/.ssh/id_rsa) | string | private_key_file = "~/.ssh/id_rsa" | -| env_type | The type of environemnt being built (only used for creating the environment) | string | env_type = "mininet" | -| mininet_ami | The AMI for the mininet environment (defaults to "ami-060d055b5ca40de8c"). Only used for running the simulator. | string | mininet_ami = "ami-060d055b5ca40de8c" | -| create_ami | When 'yes', the an EC2 image (AMI) will be created. | string | create_ami = "yes" | - -## 3. [Optional] Create an OS instance for running the mininet simulator - -This step is optional if you are running on AWS and use the AMI provided by CableLabs. - -Section 3.1 provides instructions for using Terraform to build an AMI on your AWS. - -Section 3.2 provides instructions for building an image in another environment or on baremetal. - -### 3.1. Build an AMI for running mininet on AWS - -#### 3.1.1 Create VM with Terraform - -This step will creat an VM on AWS, install all mininet dependencies and create an AMI. - -```bash -cd transparent-security/automation/env-build -terraform init -terraform apply -auto-approve -var-file="/path/to/my-mininet.tfvars" -``` - -Sample Output: - -```bash -aws_key_pair.transparent-security-mini-pk: Creating... -aws_security_group.transparent-security-img-sg: Creating... -aws_key_pair.transparent-security-mini-pk: Creation complete after 1s -. -. -. -Apply complete! Resources: 12 added, 0 changed, 0 destroyed. - -Outputs: - -ami-id = ami-0393652ac3fbc331e -ip = 34.211.114.181 -``` - -Save the ami-id and it to your variables file. - -#### 3.1.2 Remove the AMI for terraform state - -Remove the AMI from the terraform state so that it will remain after destroying the VM. - -```bash -terraform state rm aws_ami_from_instance.transparent-security-env-build -Removed aws_ami_from_instance.transparent-security-env-build -Successfully removed 1 resource instance(s). -``` - -#### 3.1.3 Clean up the VM used to create the AMI - -This step will remove everything except the AMI that was used to create the VM. - -```bash -terraform destroy -auto-approve -var-file="/path/to/my-mininet.tfvars" -``` - -Sample output: - -```bash -aws_key_pair.transparent-security-mini-pk: Refreshing state... [id=terraform-20191213203053435500000001] -aws_security_group.transparent-security-img-sg: Refreshing state... [id=sg-057e54e0162c6251a] -. -. -. -Destroy complete! Resources: 4 destroyed. -``` - -## 4. Run mininet simulator (using Terraform) - -Use the environment file created in section 2.4 - -### 4.1 Run terraform to launch the simulator on AWS. - -```bash -cd transparent-security/automation/p4/mininet -terraform init -terraform apply -auto-approve -var-file="/path/to/my-mininet.tfvars" -``` - -Sample Output: - -```bash -aws_key_pair.transparent-security-mini-pk: Creating... -aws_security_group.transparent-security-img-sg: Creating... -aws_key_pair.transparent-security-mini-pk: Creation complete after 5s -. -. -. -Apply complete! Resources: 12 added, 0 changed, 0 destroyed. - -Outputs: - -ip = 34.211.54.181 -``` - -### 4.2 Obtain Deployment Information - -```bash -# from transparent-security/automation/p4/mininet directory -terraform show -``` - -Sample output - - -```bash -# aws_instance.transparent-security-mininet-integration: -resource "aws_instance" "transparent-security-mininet-integration" { - ami = -. -. -. -. -Outputs: - -ip = "34.211.114.181" -``` - -### 4.3 SSH into EC2 Mininet VM - -Login to the VM running the simulator. Use the SSH keys indicated in the variable file to login -to the VM. - -```bash -# from transparent-security/automation/p4/mininet directory -ssh -i ubuntu@$(terraform output ip) -``` - -Sample output - - -```bash -Welcome to Ubuntu 18.04.5 LTS (GNU/Linux 4.4.0-1075-aws x86_64) - - * Documentation: https://help.ubuntu.com - * Management: https://landscape.canonical.com - * Support: https://ubuntu.com/advantage - - Get cloud support with Ubuntu Advantage Cloud Guest: - http://www.ubuntu.com/business/services/cloud - -149 packages can be updated. -89 updates are security updates. - -New release '18.04.3 LTS' available. -Run 'do-release-upgrade' to upgrade to it. - - -Last login: Wed Dec 11 22:39:13 2019 from 127.0.0.1 -ubuntu@ip-172-31-15-5:~$ -``` - -Upgrading to a newer version of Ubuntu isn't currently supported. Do so at your own risk. - -### 4.4. Development and debugging of Python - -The playbooks will be installing the python code located in the trans_sec -directory into the VM's Python runtime in place so any changes there will be -realized immediately. - -### 4.5. Cleanup the simulation environment - -This will remove the VM and other artifacts created when it was deployed. - -```bash -# from transparent-security/automation/p4/mininet directory -terraform destroy -auto-approve -var-file="/path/to/my-mininet.tfvars" -``` - -Sample output: - -```bash -. -. -Destroy complete! Resources: 12 destroyed. - -Process finished with exit code 0 -``` - -## 5. [Optional] Run mininet simulator (using ansible) - -### 5.1. Launch an AWS instance using pre-built AMI - -- From the EC2 dashboard, launch a new instance with the AMI provided by CableLabs. -- Configure the security group for the instance as follows- - -| Type | Protocol | Port Range | Source | -|:---------------:|:--------:|:----------:|:---------:| -| Custom TCP Rule | TCP | 8080 | 0.0.0.0/0 | -| SSH | TCP | 22 | 0.0.0.0/0 | -| Custom TCP Rule | TCP | 3000 | 0.0.0.0/0 | -| HTTPS | TCP | 443 | 0.0.0.0/0 | - -- Review and launch the instance. When prompted, generate a new key-pair and save it on the local machine. - -### 5.2. Create a local ansible inventory - -Generate your local ansible inventory file with the following command where - is the name of the output inventory file to be used -in step 5.4 below and is the IP of the host you want to setup. -```bash -ansible-playbook transparent-security/playbooks/mininet/local_inventory.yml \ ---extra-vars "public_ip= local_inventory=" -``` - -### 5.3. Create and inject own SSH keys - -- Login to the remote VM -```bash -ssh -i ubuntu@ -``` -- Create and inject SSH keys to be able to access the mininet hosts -```bash -ssh-keygen -t rsa -N '' -f ~/.ssh/id_rsa -touch ~/.ssh/authorized_keys -chmod 600 ~/.ssh/authorized_keys -cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys -``` - -### 5.4. Setup transparent-security directory and install dependencies on the remote VM - -- On the local machine, run the following command to setup the mininet host - -```bash -export ANSIBLE_HOST_KEY_CHECKING=False -ansible-playbook -u ubuntu -i transparent-security/playbooks/mininet/setup_host.yml --key-file ~/.ssh/id_rsa -``` - -### 5.5. Start mininet simulation - -- On the remote VM, install ansible before proceeding to begin simulation. -```bash -sudo pip install ansible -export ANSIBLE_HOST_KEY_CHECKING=False -ansible-playbook -u ubuntu -i transparent-security.ini transparent-security/playbooks/mininet/setup_mininet.yml -``` -Note - The transparent-security.ini refers to the inventory file on the remote machine which is generated in Step 5.4. - -### 5.6. Test with an attack scenario - -- On the remote VM, execute the attack scenario to validate attack detection and mitigation. -- To use the sample scenario provided by CableLabs, run the following command on the remote VM - -```bash -export ANSIBLE_HOST_KEY_CHECKING=False -ansible-playbook -u ubuntu -i transparent-security.ini transparent-security/playbooks/scenarios/full/all.yml -``` -Note - Refer the Wiki page [Attack Scenario](https://github.com/cablelabs/transparent-security/wiki/2.-Attack-Scenario) for a -detailed explanation of the attack scenario. - -## 6. [Optional] Run mininet simulator on a local VM - -For this purpose, we use a Ubuntu 18.04 VirtualBox VM running on the local machine. - -### 6.1. Setup the local VM - -- The network settings for the VM would have to allow access to the internet (to install -dependencies) and a channel for communication with the local machine. - - Bridged Adapter - - Host-only Adapter - -- Enable SSH on the VM and verify the SSH service to be active -```bash -sudo apt-get install openssh-server -sudo service ssh status -``` - -- Allow passwordless sudo by editing the /etc/sudoers file -```bash -sudo visudo -%sudo ALL=(ALL:ALL) NOPASSWD: ALL -``` -- Install Python packages -```bash -sudo apt-get update -sudo apt-get install python3.6 -y -sudo ln /usr/bin/python3.6 /usr/bin/python -``` -- Create and inject SSH keys to be able to access the mininet hosts -```bash -ssh-keygen -t rsa -N '' -f ~/.ssh/id_rsa -touch ~/.ssh/authorized_keys -chmod 600 ~/.ssh/authorized_keys -cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys -``` - -### 6.2. Setup the local machine - -- Copy the SSH key to the VM. On the local machine, -```bash -ssh-copy-id -i VM_user@VM_host -``` - -- Create a inventory file on the local machine to configure variables and VM IP address. -Note - Copy the example inventory file docs/example-local-inventory.ini to a working directory and - make changes to adapt the file to your local environment. - -### 6.3. Build the environment to run mininet simulation - -- On the local machine, run the env_build.yml to install the necessary software packages to run mininet. -```bash -export ANSIBLE_HOST_KEY_CHECKING=False; ansible-playbook -u ubuntu -i ~/variables.ini playbooks/mininet/env-build.yml -``` -Note - The env-build approximately takes 45-60 minutes to finish. - -### 6.4. Setup transparent-security directory and install dependencies on the VM - - - On the local machine, run the following command to create the inventory file - for setting up the mininet host: - ```bash -ansible-playbook transparent-security/playbooks/mininet/local_inventory.yml \ ---extra-vars "public_ip= local_inventory=" - ``` - - - On the local machine, run the following command to setup the mininet host: - ```bash - export ANSIBLE_HOST_KEY_CHECKING=False - ansible-playbook -u ubuntu -i transparent-security/playbooks/mininet/setup_host.yml --key-file ~/.ssh/id_rsa - ``` -### 6.5. Start mininet simulation - -- On the VM, install ansible before proceeding to begin simulation. -```bash -sudo pip install ansible -export ANSIBLE_HOST_KEY_CHECKING=False -ansible-playbook -u ubuntu -i transparent-security.ini transparent-security/playbooks/mininet/setup_mininet.yml -``` -Note - The transparent-security.ini refers to the inventory file on the remote machine which is generated in Step 6.4. - -### 6.6. Test with the UDP Flood attack scenario - -- On the VM, execute the attack scenario to validate attack detection and mitigation. -- To use the sample scenario provided by CableLabs, run the following command on the remote VM - -```bash -export ANSIBLE_HOST_KEY_CHECKING=False -ansible-playbook -u ubuntu -i transparent-security.ini transparent-security/playbooks/scenarios/full/all.yml -``` -Note - Refer the Wiki page [Attack Scenario](https://github.com/cablelabs/transparent-security/wiki/2.-Attack-Scenario) for a -detailed explanation of the attack scenario. - -## 7. Using Mininet - -- Refer the Wiki page [Using Mininet](https://github.com/cablelabs/transparent-security/wiki/3.-Using-Mininet) for -information on the default mininet architecture, how to access devices and capture packet-level data. diff --git a/docs/SIDDHI_AE_SETUP.md b/docs/ae/SIDDHI_AE_SETUP.md similarity index 100% rename from docs/SIDDHI_AE_SETUP.md rename to docs/ae/SIDDHI_AE_SETUP.md diff --git a/docs/mininet-example.tfvars b/docs/mininet-example.tfvars deleted file mode 100644 index e6e6bc4f..00000000 --- a/docs/mininet-example.tfvars +++ /dev/null @@ -1,27 +0,0 @@ -#Unique ID for this build -build_id = "changeme-1" - -# EC2 credentials -# NOTE-The mininet-ami variable MUST be changed to point to the image created from the CI env-build -# Amazon EC2 access key -access_key = "AKIAIOSFODNN7EXAMPLE" - -# Amazon EC2 secret key -secret_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" - -# Amazon EC2 region; Optional to change -ec2_region = "us-west-2" - -# Used to inject into the VM for SSH access with the user'ubuntu' (defaults to ~/.ssh/id_rsa.pub) -# public_key_file = "~/.ssh/id_rsa.pub" - -# Used to access the VM via SSH with the user 'ubuntu' (defaults to ~/.ssh/id_rsa) -# private_key_file = "~/.ssh/id_rsa" - -# The type of environemnt being used -# Only used for creating the environment -# env_type = "mininet" - -# MUST be changed to the AMI ID generated from CI env-build -# Only used for running simulator -# mininet_ami = "ami-060d055b5ca40de8c" diff --git a/p4/aggregate/aggregate.p4 b/p4/aggregate/aggregate.p4 deleted file mode 100755 index 9af53dc1..00000000 --- a/p4/aggregate/aggregate.p4 +++ /dev/null @@ -1,273 +0,0 @@ -/* -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -*/ -/* -*- P4_16 -*- */ -#include - -/* TPS includes */ -#include "../include/tps_consts.p4" -#include "../include/tps_headers.p4" -#include "../include/tps_parser.p4" -#include "../include/tps_checksum.p4" -#include "../include/tps_egress.p4" - -/************************************************************************* -************** I N G R E S S P R O C E S S I N G ******************** -*************************************************************************/ - -control TpsAggIngress(inout headers hdr, - inout metadata meta, - inout standard_metadata_t standard_metadata) { - - counter(MAX_DEVICE_ID, CounterType.packets_and_bytes) forwardedPackets; - - action pack_meta_tcp() { - meta.dst_port = hdr.tcp.dst_port; - } - - action pack_meta_udp() { - meta.dst_port = hdr.udp_int.dst_port; - } - - action pack_meta_ipv4() { - meta.ipv6_addr = 0; - meta.ipv4_addr = hdr.ipv4.dstAddr; - } - - action pack_meta_ipv6() { - meta.ipv4_addr = 0; - meta.ipv6_addr = hdr.ipv6.dstAddr; - } - - action data_drop() { - mark_to_drop(standard_metadata); - } - - table data_drop_t { - key = { - hdr.ethernet.src_mac: exact; - meta.ipv4_addr: exact; - meta.ipv6_addr: exact; - meta.dst_port: exact; - } - actions = { - data_drop; - NoAction; - } - size = TABLE_SIZE; - default_action = NoAction(); - } - - action data_forward(egressSpec_t port) { - standard_metadata.egress_spec = port; - } - - table data_forward_t { - key = { - hdr.ethernet.dst_mac: exact; - } - actions = { - data_forward; - NoAction; - } - size = TABLE_SIZE; - default_action = NoAction(); - } - - action add_switch_id(bit<32> switch_id) { - hdr.int_meta_2.setValid(); - hdr.ipv4.totalLen = hdr.ipv4.totalLen + BYTES_PER_SHIM * INT_SHIM_HOP_SIZE; - hdr.udp_int.len = hdr.udp_int.len + SWITCH_ID_HDR_BYTES; - hdr.ipv6.payload_len = hdr.ipv6.payload_len + BYTES_PER_SHIM * INT_SHIM_HOP_SIZE; - hdr.int_shim.length = hdr.int_shim.length + INT_SHIM_HOP_SIZE; - hdr.int_header.remaining_hop_cnt = hdr.int_header.remaining_hop_cnt - 1; - hdr.int_meta_2.switch_id = switch_id; - } - - table add_switch_id_t { - key = { - hdr.udp_int.dst_port: exact; - } - actions = { - add_switch_id; - NoAction; - } - size = TABLE_SIZE; - default_action = NoAction(); - } - - action data_inspect_packet(bit<32> device, bit<32> switch_id) { - hdr.int_shim.setValid(); - hdr.int_header.setValid(); - hdr.int_meta.setValid(); - - hdr.int_shim.npt = INT_SHIM_NPT_UDP_FULL_WRAP; - hdr.int_shim.type = INT_SHIM_TYPE; - hdr.int_shim.length = INT_SHIM_BASE_SIZE; - - hdr.int_header.ver = INT_VERSION; - hdr.int_header.domain_id = INT_SHIM_DOMAIN_ID; - hdr.int_header.meta_len = INT_META_LEN; - hdr.int_header.instr_bit_0 = TRUE; - hdr.int_header.ds_instr_0 = TRUE; - hdr.int_header.ds_flags_1 = TRUE; - hdr.int_header.remaining_hop_cnt = MAX_HOPS; - - hdr.int_meta.switch_id = switch_id; - hdr.int_meta.orig_mac = hdr.ethernet.src_mac; - - forwardedPackets.count(device); - } - - action data_inspect_packet_ipv4() { - hdr.ipv4.ttl = hdr.ipv4.ttl - 1; - hdr.int_shim.next_proto = hdr.ipv4.protocol; - hdr.ipv4.protocol = TYPE_UDP; - hdr.ipv4.totalLen = hdr.ipv4.totalLen + ((bit<16>)hdr.int_shim.length * BYTES_PER_SHIM * INT_SHIM_HOP_SIZE) + UDP_HDR_BYTES; - } - - action data_inspect_packet_ipv6() { - hdr.int_shim.next_proto = hdr.ipv6.next_hdr_proto; - hdr.ipv6.next_hdr_proto = TYPE_UDP; - hdr.ipv6.payload_len = hdr.ipv6.payload_len + IPV6_HDR_BYTES + ((bit<16>)hdr.int_shim.length * BYTES_PER_SHIM * INT_SHIM_HOP_SIZE) + UDP_HDR_BYTES; - } - - table data_inspection_t { - key = { - hdr.ethernet.src_mac: exact; - } - actions = { - data_inspect_packet; - NoAction; - } - size = TABLE_SIZE; - default_action = NoAction(); - } - - action insert_udp_int_for_udp() { - hdr.udp.setValid(); - hdr.udp.src_port = hdr.udp_int.src_port; - hdr.udp.dst_port = hdr.udp_int.dst_port; - hdr.udp.len = hdr.udp_int.len; - hdr.udp_int.src_port = UDP_INT_SRC_PORT; - hdr.udp_int.dst_port = UDP_INT_DST_PORT; - hdr.udp_int.len = hdr.udp.len + ((bit<16>)hdr.int_shim.length * BYTES_PER_SHIM * INT_SHIM_HOP_SIZE) + UDP_HDR_BYTES; - } - - action insert_udp_int_for_tcp_ipv4() { - hdr.udp_int.setValid(); - hdr.udp_int.src_port = UDP_INT_SRC_PORT; - hdr.udp_int.dst_port = UDP_INT_DST_PORT; - hdr.udp_int.len = hdr.ipv4.totalLen - IPV4_HDR_BYTES; - } - - action insert_udp_int_for_tcp_ipv6() { - hdr.udp_int.setValid(); - hdr.udp_int.src_port = UDP_INT_SRC_PORT; - hdr.udp_int.dst_port = UDP_INT_DST_PORT; - hdr.udp_int.len = hdr.ipv6.payload_len - IPV6_HDR_BYTES; - } - - action control_drop() { - mark_to_drop(standard_metadata);; - } - - action generate_learn_notification() { - digest((bit<32>) 1024, - { hdr.arp.src_mac, - standard_metadata.ingress_port - }); - } - - action arp_flood() { - standard_metadata.mcast_grp = 1; - } - - table arp_flood_t { - key = { - hdr.ethernet.dst_mac: exact; - } - actions = { - arp_flood; - NoAction; - } - default_action = NoAction(); - } - - apply { - if (hdr.arp.isValid()) { - generate_learn_notification(); - if (hdr.arp.opcode == 1) { - arp_flood_t.apply(); - } else if (hdr.arp.opcode == 2) { - data_forward_t.apply(); - } - } else if (standard_metadata.egress_spec != DROP_PORT) { - if (hdr.int_shim.isValid()) { - // Add switch ID into existing INT data - add_switch_id_t.apply(); - } else { - // Pack metadata with original header values for data drop - if (hdr.udp_int.isValid()) { - pack_meta_udp(); - } else if (hdr.tcp.isValid()) { - pack_meta_tcp(); - } - if (hdr.ipv4.isValid()) { - pack_meta_ipv4(); - } else if (hdr.ipv6.isValid()) { - pack_meta_ipv6(); - } - - // Check to see if need to add INT - data_inspection_t.apply(); - - // Add IP & Protocol specific data to new INT data - if (hdr.int_shim.isValid()) { - if (hdr.ipv4.isValid()) { - data_inspect_packet_ipv4(); - if (hdr.udp_int.isValid()) { - insert_udp_int_for_udp(); - } else if (hdr.tcp.isValid()) { - insert_udp_int_for_tcp_ipv4(); - } - } - else if (hdr.ipv6.isValid()) { - data_inspect_packet_ipv6(); - if (hdr.udp_int.isValid()) { - insert_udp_int_for_udp(); - } else if (hdr.tcp.isValid()) { - insert_udp_int_for_tcp_ipv6(); - } - } - } - } - data_forward_t.apply(); - } - data_drop_t.apply(); - } -} - -/************************************************************************* -*********************** S W I T C H ************************************ -*************************************************************************/ - -V1Switch( - TpsAggParser(), - TpsVerifyChecksum(), - TpsAggIngress(), - TpsEgress(), - TpsComputeChecksum(), - TpsAggDeparser() -) main; diff --git a/p4/core/core.p4 b/p4/core/core.p4 deleted file mode 100755 index 6aa3ce52..00000000 --- a/p4/core/core.p4 +++ /dev/null @@ -1,336 +0,0 @@ -/* -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -*/ -/* -*- P4_16 arch -*- */ -#include - -/* TPS includes */ -#include "../include/tps_consts.p4" -#include "../include/tps_headers.p4" -#include "../include/tps_checksum.p4" -#include "../include/tps_parser.p4" -#include "../include/tps_egress.p4" - -const bit<32> INT_CTR_SIZE = 1; - -/************************************************************************* -************** I N G R E S S P R O C E S S I N G ******************** -*************************************************************************/ - -control TpsCoreIngress(inout headers hdr, - inout metadata meta, - inout standard_metadata_t standard_metadata) { - - /** - * Responsible for recirculating a packet after egress processing - */ - action recirculate_packet() { - recirculate(standard_metadata); - } - - /** - * Responsible for cloning a packet as ingressed - */ - action clone_packet_i2e() { - clone3(CloneType.I2E, I2E_CLONE_SESSION_ID, standard_metadata); - } - - /** - * Adds INT data if data_inspection_t table has a match on hdr.ethernet.src_mac - */ - action data_inspect_packet(bit<32> switch_id) { - hdr.int_meta_3.setValid(); - hdr.int_shim.length = hdr.int_shim.length + INT_SHIM_HOP_SIZE; - hdr.ipv4.totalLen = hdr.ipv4.totalLen + (INT_SHIM_HOP_SIZE * BYTES_PER_SHIM); - hdr.udp_int.len = hdr.udp_int.len + (INT_SHIM_HOP_SIZE * BYTES_PER_SHIM); - hdr.int_header.remaining_hop_cnt = hdr.int_header.remaining_hop_cnt - 1; - hdr.int_meta_3.switch_id = switch_id; - } - - table data_inspection_t { - key = { - hdr.udp_int.dst_port: exact; - } - actions = { - data_inspect_packet; - NoAction; - } - size = TABLE_SIZE; - default_action = NoAction(); - } - - /** - * Prepares a packet to be forwarded when data_forward tables have a match on dstAddr - */ - action data_forward(egressSpec_t port) { - standard_metadata.egress_spec = port; - } - - table data_forward_t { - key = { - hdr.ethernet.dst_mac: exact; - } - actions = { - data_forward; - NoAction; - } - size = TABLE_SIZE; - default_action = NoAction(); - } - - action arp_forward(egressSpec_t port) { - standard_metadata.egress_spec = port; - } - - table arp_forward_t { - key = { - hdr.ethernet.dst_mac: exact; - } - actions = { - arp_forward; - NoAction; - } - size = TABLE_SIZE; - default_action = NoAction(); - } - - /** - * Removes INT data from a packet - */ - action clear_int() { - hdr.ipv4.protocol = hdr.int_shim.next_proto; - hdr.ipv6.next_hdr_proto = hdr.int_shim.next_proto; - hdr.ipv4.totalLen = hdr.ipv4.totalLen - ((bit<16>)hdr.int_shim.length * BYTES_PER_SHIM * INT_SHIM_HOP_SIZE) - UDP_HDR_BYTES; - hdr.ipv6.payload_len = hdr.ipv6.payload_len - ((bit<16>)hdr.int_shim.length * BYTES_PER_SHIM * INT_SHIM_HOP_SIZE); - - hdr.udp_int.setInvalid(); - hdr.int_shim.setInvalid(); - hdr.int_header.setInvalid(); - hdr.int_meta_2.setInvalid(); - hdr.int_meta_3.setInvalid(); - hdr.int_meta.setInvalid(); - } - - action generate_learn_notification() { - digest((bit<32>) 1024, - { hdr.arp.src_mac, - standard_metadata.ingress_port - }); - } - - action arp_flood() { - standard_metadata.mcast_grp = 1; - } - - table arp_flood_t { - key = { - hdr.ethernet.dst_mac: exact; - } - actions = { - arp_flood; - NoAction; - } - default_action = NoAction(); - } - - apply { - if (hdr.arp.isValid()) { - generate_learn_notification(); - if (hdr.arp.opcode == 1) { - arp_flood_t.apply(); - } - else if (hdr.arp.opcode == 2) { - arp_forward_t.apply(); - } - } else if (standard_metadata.egress_spec != DROP_PORT) { - if (IS_NORMAL(standard_metadata)) { - // First pass - data_inspection_t.apply(); - recirculate_packet(); - } else if (IS_RECIRCULATED(standard_metadata)) { - // second pass - data_forward_t.apply(); - if (hdr.int_shim.isValid()) { - clone_packet_i2e(); - clear_int(); - } - } - } - } -} - - -/************************************************************************* -**************** E G R E S S P R O C E S S I N G ******************** -*************************************************************************/ - -control TpsCoreEgress(inout headers hdr, - inout metadata meta, - inout standard_metadata_t standard_metadata) { - - register>(INT_CTR_SIZE) trpt_pkts; - - action control_drop() { - mark_to_drop(standard_metadata); - } - - /** - * Restrutures data within INT packet into a Telemetry Report packet type for ipv4 - */ - action init_telem_rpt() { - hdr.trpt_eth.setValid(); - hdr.trpt_udp.setValid(); - hdr.trpt_hdr.setValid(); - - hdr.trpt_hdr.node_id = hdr.int_meta_3.switch_id; - hdr.trpt_hdr.rep_type = TRPT_RPT_TYPE_INT_2; - hdr.trpt_hdr.in_type = TRPT_HDR_IN_TYPE_ETH; - hdr.trpt_hdr.md_len = hdr.int_shim.length; - - hdr.trpt_eth.dst_mac = hdr.ethernet.dst_mac; - hdr.trpt_eth.src_mac = hdr.ethernet.src_mac; - - hdr.trpt_udp.dst_port = TRPT_INT_DST_PORT; - - hdr.trpt_hdr.ver = TRPT_VERSION; - hdr.trpt_hdr.domain_id = TRPT_HDR_DOMAIN_ID; - - hdr.trpt_hdr.sequence_no = 0; - hdr.trpt_hdr.sequence_pad = 0; - - hdr.trpt_hdr.rpt_len = hdr.trpt_hdr.rpt_len + hdr.int_shim.length + 5; /* 5 Reflects TRPT ethernet & udp packets - - /* TODO - determine if counter resets to 0 once it reaches max */ - trpt_pkts.read(hdr.trpt_hdr.sequence_no, INT_CTR_SIZE - 1); - hdr.trpt_hdr.sequence_no = hdr.trpt_hdr.sequence_no + 1; - //trpt_pkts.write(INT_CTR_SIZE - 1, hdr.trpt_hdr.sequence_no); - } - - /* Sets the trpt_eth.in_type for IPv4 */ - action set_telem_rpt_in_type_ipv4() { - hdr.trpt_hdr.in_type = TRPT_HDR_IN_TYPE_IPV4; - } - - /* Sets the trpt_eth.in_type for IPv6 */ - action set_telem_rpt_in_type_ipv6() { - hdr.trpt_hdr.in_type = TRPT_HDR_IN_TYPE_IPV6; - } - - /** - * Restrutures data within INT packet into a Telemetry Report packet type for ipv4 - */ - action setup_telem_rpt_ipv4(ip4Addr_t ae_ip) { - hdr.trpt_ipv4.setValid(); - - hdr.trpt_eth.etherType = TYPE_IPV4; - - hdr.trpt_ipv4.version = 0x4; - hdr.trpt_ipv4.ihl = 0x5; - hdr.trpt_ipv4.ttl = DFLT_IPV4_TTL; - hdr.trpt_ipv4.flags = IPV4_DONT_FRAGMENT; - hdr.trpt_ipv4.protocol = TYPE_UDP; - hdr.trpt_ipv4.srcAddr = hdr.ipv4.srcAddr; - hdr.trpt_ipv4.dstAddr = ae_ip; - hdr.trpt_udp.len = hdr.trpt_ipv4.totalLen - IPV4_HDR_BYTES; - hdr.trpt_ipv4.totalLen = (bit<16>)standard_metadata.packet_length + IPV4_HDR_BYTES + UDP_HDR_BYTES + TRPT_HDR_BASE_BYTES; - } - - /** - * Restrutures data within INT packet into a Telemetry Report packet type for ipv4 - */ - action setup_telem_rpt_ipv6(ip6Addr_t ae_ip) { - hdr.trpt_ipv6.setValid(); - - hdr.trpt_eth.etherType = TYPE_IPV6; - - hdr.trpt_ipv6.next_hdr_proto = TYPE_UDP; - hdr.trpt_ipv6.srcAddr = hdr.ipv6.srcAddr; - hdr.trpt_ipv6.dstAddr = ae_ip; - } - - /** - * Updates the TRPT header length value when the underlying packet is IPv4 - */ - action update_trpt_hdr_len_ipv4() { - hdr.trpt_hdr.rpt_len = hdr.trpt_hdr.rpt_len + 5; - /*hdr.trpt_udp.len = hdr.ipv4.totalLen + IPV4_HDR_BYTES + TRPT_HDR_BYTES + 2;*/ - hdr.trpt_udp.len = hdr.ipv4.totalLen + IPV4_HDR_BYTES + TRPT_HDR_BYTES - 4; - hdr.trpt_ipv4.totalLen = (bit<16>)standard_metadata.packet_length + IPV4_HDR_BYTES + UDP_HDR_BYTES + TRPT_HDR_BASE_BYTES; - } - - /** - * Updates the TRPT header length value when the underlying packet is IPv6 - */ - action update_trpt_hdr_len_ipv6() { - hdr.trpt_hdr.rpt_len = hdr.trpt_hdr.rpt_len + 10; - hdr.trpt_udp.len = hdr.ipv6.payload_len + IPV6_HDR_BYTES + TRPT_HDR_BYTES - ETH_HDR_BYTES; - hdr.trpt_ipv6.payload_len = (bit<16>)standard_metadata.packet_length + IPV6_HDR_BYTES + UDP_HDR_BYTES + TRPT_HDR_BASE_BYTES; - } - - /* TODO - Design table properly, currently just making IPv4 or IPv6 Choices */ - table setup_telemetry_rpt_t { - key = { - hdr.udp_int.dst_port: exact; - } - actions = { - setup_telem_rpt_ipv4; - setup_telem_rpt_ipv6; - control_drop; - } - size = TABLE_SIZE; - default_action = control_drop(); - } - - apply { - if (hdr.arp.isValid()) { - if(IS_REPLICATED(standard_metadata)) { - if (standard_metadata.egress_port == standard_metadata.ingress_port) { - control_drop(); - } - } - } else if (standard_metadata.egress_spec != DROP_PORT) { - if (IS_I2E_CLONE(standard_metadata)) { - init_telem_rpt(); - if (hdr.ipv4.isValid()) { - set_telem_rpt_in_type_ipv4(); - } - if (hdr.ipv6.isValid()) { - set_telem_rpt_in_type_ipv6(); - } - setup_telemetry_rpt_t.apply(); - if (hdr.ipv4.isValid()) { - update_trpt_hdr_len_ipv4(); - } else if (hdr.ipv6.isValid()) { - update_trpt_hdr_len_ipv6(); - } - /* Ensure packet is no larger than TRPT_MAX_BYTES */ - truncate(TRPT_MAX_BYTES); - } - } - } -} - - -/************************************************************************* -*********************** S W I T C H ************************************ -*************************************************************************/ - -V1Switch( - TpsCoreParser(), - TpsVerifyChecksum(), - TpsCoreIngress(), - TpsCoreEgress(), - TpsComputeChecksum(), - TpsDeparser() -) main; diff --git a/p4/gateway/gateway.p4 b/p4/gateway/gateway.p4 deleted file mode 100755 index c2001f6f..00000000 --- a/p4/gateway/gateway.p4 +++ /dev/null @@ -1,360 +0,0 @@ -/* -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -*/ -/* -*- P4_16 -*- */ -#include - -/* TPS includes */ -#include "../include/tps_consts.p4" -#include "../include/tps_headers.p4" -#include "../include/tps_parser.p4" -#include "../include/tps_checksum.p4" -#include "../include/tps_egress.p4" - -/************************************************************************* -************** I N G R E S S P R O C E S S I N G ******************** -*************************************************************************/ - -control TpsGwIngress(inout headers hdr, - inout metadata meta, - inout standard_metadata_t standard_metadata) { - - // TODO/FIXME - so this works for both BMV2 & TOFINO - #ifdef BMV2 - counter(MAX_DEVICE_ID, CounterType.packets_and_bytes) forwardedPackets; - counter(MAX_DEVICE_ID, CounterType.packets_and_bytes) droppedPackets; - #endif - - action data_forward(egressSpec_t port, bit <48> switch_mac) { - standard_metadata.egress_spec = port; - hdr.ethernet.src_mac = switch_mac; - } - - table data_forward_t { - key = { - hdr.ethernet.dst_mac: exact; - } - actions = { - data_forward; - NoAction; - } - size = TABLE_SIZE; - default_action = NoAction(); - } - - action data_inspect_packet(bit<32> device, bit<32> switch_id) { - hdr.udp_int.setValid(); - hdr.int_shim.setValid(); - hdr.int_header.setValid(); - hdr.int_meta.setValid(); - - hdr.udp_int.dst_port = UDP_INT_DST_PORT; - - hdr.int_shim.npt = INT_SHIM_NPT_UDP_FULL_WRAP; - hdr.int_shim.type = INT_SHIM_TYPE; - hdr.int_shim.length = INT_SHIM_BASE_SIZE; - - hdr.int_header.ver = INT_VERSION; - hdr.int_header.domain_id = INT_SHIM_DOMAIN_ID; - hdr.int_header.meta_len = INT_META_LEN; - hdr.int_header.instr_bit_0 = TRUE; - hdr.int_header.ds_instr_0 = TRUE; - hdr.int_header.ds_flags_1 = TRUE; - hdr.int_header.remaining_hop_cnt = MAX_HOPS; - - hdr.int_meta.switch_id = switch_id; - hdr.int_meta.orig_mac = hdr.ethernet.src_mac; - - #ifdef BMV2 - forwardedPackets.count(device); - #endif - } - - action data_inspect_packet_ipv4() { - hdr.ipv4.ttl = hdr.ipv4.ttl - 1; - hdr.int_shim.next_proto = hdr.ipv4.protocol; - hdr.ipv4.protocol = TYPE_UDP; - - #ifdef BMV2 - hdr.ipv4.totalLen = hdr.ipv4.totalLen + ((bit<16>)hdr.int_shim.length * BYTES_PER_SHIM * INT_SHIM_HOP_SIZE) + UDP_HDR_BYTES; - #endif - } - - action data_inspect_packet_ipv6() { - hdr.ipv6.next_hdr_proto = TYPE_UDP; - hdr.int_shim.next_proto = hdr.ipv6.next_hdr_proto; - - #ifdef BMV2 - hdr.ipv6.payload_len = hdr.ipv6.payload_len + IPV6_HDR_BYTES + ((bit<16>)hdr.int_shim.length * BYTES_PER_SHIM * INT_SHIM_HOP_SIZE) + UDP_HDR_BYTES; - #endif - } - - table data_inspection_t { - key = { - hdr.ethernet.src_mac: exact; - } - actions = { - data_inspect_packet; - NoAction; - } - size = TABLE_SIZE; - default_action = NoAction(); - } - - action insert_udp_int_for_udp() { - hdr.udp_int.len = hdr.udp.len + ((bit<16>)hdr.int_shim.length * BYTES_PER_SHIM * INT_SHIM_HOP_SIZE) + UDP_HDR_BYTES; - } - - action insert_udp_int_for_tcp() { - hdr.udp_int.len = ((bit<16>)hdr.int_shim.length * BYTES_PER_SHIM * INT_SHIM_HOP_SIZE) + TCP_HDR_BYTES + UDP_HDR_BYTES; - } - - action data_drop(bit<32> device) { - mark_to_drop(standard_metadata); - #ifdef BMV2 - droppedPackets.count(device); - #endif - } - - table data_drop_udp_ipv4_t { - key = { - hdr.ethernet.src_mac: exact; - hdr.ipv4.dstAddr: exact; - hdr.udp.dst_port: exact; - } - actions = { - data_drop; - NoAction; - } - size = TABLE_SIZE; - default_action = NoAction(); - } - - table data_drop_udp_ipv6_t { - key = { - hdr.ethernet.src_mac: exact; - hdr.ipv6.dstAddr: exact; - hdr.udp.dst_port: exact; - } - actions = { - data_drop; - NoAction; - } - size = TABLE_SIZE; - default_action = NoAction(); - } - - table data_drop_tcp_ipv4_t { - key = { - hdr.ethernet.src_mac: exact; - hdr.ipv4.dstAddr: exact; - hdr.tcp.dst_port: exact; - } - actions = { - data_drop; - NoAction; - } - size = TABLE_SIZE; - default_action = NoAction(); - } - - table data_drop_tcp_ipv6_t { - key = { - hdr.ethernet.src_mac: exact; - hdr.ipv6.dstAddr: exact; - hdr.tcp.dst_port: exact; - } - actions = { - data_drop; - NoAction; - } - size = TABLE_SIZE; - default_action = NoAction(); - } - - action control_drop() { - mark_to_drop(standard_metadata);; - } - - action generate_learn_notification() { - digest((bit<32>) 1024, - { hdr.arp.src_mac, - standard_metadata.ingress_port - }); - } - - action nat_learn_notification() { - digest((bit<32>) 1024, - { hdr.udp.src_port, - hdr.tcp.src_port, - hdr.ipv4.srcAddr - }); - } - - action udp_global_to_local(bit <16> dst_port, bit< 32> ip_dstAddr) { - hdr.udp.dst_port = dst_port; - hdr.ipv4.dstAddr = ip_dstAddr; - } - - table udp_global_to_local_t { - key = { - hdr.udp.dst_port: exact; - hdr.ipv4.dstAddr: lpm; - } - actions = { - udp_global_to_local; - NoAction; - } - default_action = NoAction(); - } - - action tcp_global_to_local(bit <16> dst_port, bit< 32> ip_dstAddr) { - hdr.tcp.dst_port = dst_port; - hdr.ipv4.dstAddr = ip_dstAddr; - } - - table tcp_global_to_local_t { - key = { - hdr.tcp.dst_port: exact; - hdr.ipv4.dstAddr: lpm; - } - actions = { - tcp_global_to_local; - NoAction; - } - default_action = NoAction(); - } - - action udp_local_to_global(bit <16> src_port, bit <32> ip_srcAddr) { - hdr.udp.src_port = src_port; - hdr.ipv4.srcAddr = ip_srcAddr; - } - - table udp_local_to_global_t { - key = { - hdr.udp.src_port: exact; - hdr.ipv4.srcAddr: lpm; - } - actions = { - udp_local_to_global; - NoAction; - } - default_action = NoAction(); - } - - action tcp_local_to_global(bit <16> src_port, bit< 32> ip_srcAddr) { - hdr.tcp.src_port = src_port; - hdr.ipv4.srcAddr = ip_srcAddr; - } - - table tcp_local_to_global_t { - key = { - hdr.tcp.src_port: exact; - hdr.ipv4.srcAddr: lpm; - } - actions = { - tcp_local_to_global; - NoAction; - } - default_action = NoAction(); - } - - action mac_lookup(macAddr_t dst_mac) { - hdr.ethernet.dst_mac = dst_mac; - } - - table mac_lookup_ipv4_t { - key = { - hdr.ipv4.dstAddr: lpm; - } - actions = { - mac_lookup; - NoAction; - } - default_action = NoAction(); - } - - table mac_lookup_ipv6_t { - key = { - hdr.ipv6.dstAddr: lpm; - } - actions = { - mac_lookup; - NoAction; - } - default_action = NoAction(); - } - - apply { - if (hdr.udp.isValid()) { - if (hdr.ipv4.isValid()) { - data_drop_udp_ipv4_t.apply(); - } - if (hdr.ipv6.isValid()) { - data_drop_udp_ipv6_t.apply(); - } - } else if (hdr.tcp.isValid()) { - if (hdr.ipv4.isValid()) { - data_drop_tcp_ipv4_t.apply(); - } - if (hdr.ipv6.isValid()) { - data_drop_tcp_ipv6_t.apply(); - } - } - if (hdr.arp.isValid()) { - generate_learn_notification(); - } - else if (standard_metadata.egress_spec != DROP_PORT) { - data_inspection_t.apply(); - - if (hdr.int_shim.isValid()) { - if (hdr.ipv4.isValid()) { - data_inspect_packet_ipv4(); - } - if (hdr.ipv6.isValid()) { - data_inspect_packet_ipv6(); - } - #ifdef BMV2 - nat_learn_notification(); - if (hdr.udp.isValid()) { - udp_local_to_global_t.apply(); - udp_global_to_local_t.apply(); - insert_udp_int_for_udp(); - } - else if (hdr.tcp.isValid()) { - tcp_local_to_global_t.apply(); - tcp_global_to_local_t.apply(); - insert_udp_int_for_tcp(); - } - #endif - } - mac_lookup_ipv4_t.apply(); - mac_lookup_ipv6_t.apply(); - data_forward_t.apply(); - } - } -} - -/************************************************************************* -*********************** S W I T C H ************************************ -*************************************************************************/ - -V1Switch( - TpsGwParser(), - TpsVerifyChecksum(), - TpsGwIngress(), - TpsEgress(), - TpsComputeChecksum(), - TpsDeparser() -) main; diff --git a/playbooks/dependencies/bmv2.yml b/playbooks/dependencies/bmv2.yml deleted file mode 100644 index e96ebf4a..00000000 --- a/playbooks/dependencies/bmv2.yml +++ /dev/null @@ -1,155 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -- hosts: all - gather_facts: no - tasks: - - name: Update cache before installing python3-pip (apt task does not working here) - become: yes - command: apt update - - - name: install packages for behavioral-model - become: yes - apt: - update_cache: yes - name: - - python3-pip - - git - - autoconf - - automake - - cmake - - libjudy-dev - - libgmp-dev - - libpcap-dev - - libboost-dev - - libboost-test-dev - - libboost-program-options-dev - - libboost-system-dev - - libboost-filesystem-dev - - libboost-thread-dev - - libevent-dev - - libtool - - flex - - bison - - pkg-config - - curl - - make - - g++ - - libreadline-dev - - - name: clone behavioral-model - git: - repo: https://github.com/p4lang/behavioral-model.git - dest: ~/behavioral-model - version: "{{ bm_version }}" - - - name: Install behavioral-model dependencies - command: ./install_deps.sh - args: - chdir: ~/behavioral-model - register: cmd_out - changed_when: cmd_out is not failed - async: 600 - ignore_errors: yes - - - name: Upgrade cffi python package for behavioral-model nmpy - become: yes - pip: - name: - - cffi==1.5.2 - when: cmd_out is failed - - - name: Install behavioral-model dependencies - command: ./install_deps.sh - args: - chdir: ~/behavioral-model - register: second_out - changed_when: second_out is not failed - async: 600 - when: cmd_out is failed - -- import_playbook: p4runtime.yml - -- hosts: all - gather_facts: no - tasks: - - name: autogen behavioral-model - command: ./autogen.sh - args: - chdir: ~/behavioral-model - register: cmd_out - changed_when: cmd_out is not failed - - - name: configure behavioral-model - command: ./configure --enable-debugger --with-pi - args: - chdir: ~/behavioral-model - register: cmd_out - changed_when: cmd_out is not failed - - - name: make behavioral-model - command: make -j4 - args: - chdir: ~/behavioral-model - register: cmd_out - changed_when: cmd_out is not failed - async: 1200 - - - name: install behavioral-model - command: sudo make install - args: - chdir: ~/behavioral-model - register: cmd_out - changed_when: cmd_out is not failed - - - name: ldconfig behavioral-model - command: sudo ldconfig - args: - chdir: ~/behavioral-model - register: cmd_out - changed_when: cmd_out is not failed - - - name: autogen behavioral-model target simple_switch_grpc - command: ./autogen.sh - args: - chdir: ~/behavioral-model/targets/simple_switch_grpc - register: cmd_out - changed_when: cmd_out is not failed - - - name: configure behavioral-model target simple_switch_grpc - command: ./configure --with-pi - args: - chdir: ~/behavioral-model/targets/simple_switch_grpc - register: cmd_out - changed_when: cmd_out is not failed - - - name: make behavioral-model target simple_switch_grpc - command: make -j4 - args: - chdir: ~/behavioral-model/targets/simple_switch_grpc - register: cmd_out - changed_when: cmd_out is not failed - - - name: install behavioral-model simple_switch_grpc - command: sudo make install - args: - chdir: ~/behavioral-model/targets/simple_switch_grpc - register: cmd_out - changed_when: cmd_out is not failed - - - name: ldconfig behavioral-model target simple_switch_grpc - command: sudo ldconfig - args: - chdir: ~/behavioral-model/targets/simple_switch_grpc - register: cmd_out - changed_when: cmd_out is not failed diff --git a/playbooks/dependencies/grpc.yml b/playbooks/dependencies/grpc.yml deleted file mode 100644 index 51f13732..00000000 --- a/playbooks/dependencies/grpc.yml +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -- hosts: all - - gather_facts: no - - environment: - LDFLAGS: "-Wl,-s" - - tasks: - - name: apt update - become: yes - apt: - update_cache: yes - - - name: install packages for grpc - become: yes - apt: - name: - - python3-pip - - git - - autoconf - - automake - - libtool - - curl - - make - - g++ - - - name: pip install cygrpc and upgrade setuptools - become: yes - pip: - name: - - cygrpc - - setuptools>=40.3.0 - when: ubuntu_version == "18" - - - name: clone grpc - git: - repo: https://github.com/grpc/grpc - dest: ~/grpc - version: "{{ grpc_version }}" - force: yes - - - name: build grpc - command: make -j4 - args: - chdir: ~/grpc - register: cmd_out - changed_when: cmd_out is not failed - - - name: configure grpc - command: sudo make install - args: - chdir: ~/grpc - register: cmd_out - changed_when: cmd_out is not failed - - - name: ldconfig grpc - command: sudo ldconfig - args: - chdir: ~/grpc - register: cmd_out - changed_when: cmd_out is not failed - - - name: Copy Python {{ python_version }} site-packages to dist-packages PI - command: "sudo cp -r /home/ubuntu/grpc/src/python/grpcio /usr/local/lib/python{{ python_version }}/dist-packages" diff --git a/playbooks/dependencies/mininet.yml b/playbooks/dependencies/mininet.yml deleted file mode 100644 index 367070ac..00000000 --- a/playbooks/dependencies/mininet.yml +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -- hosts: all - - gather_facts: no - - tasks: - - name: Update cache before installing python3-pip (apt task does not working here) - become: yes - command: apt update - - - name: apt install packages for mininet - become: yes - apt: - update_cache: yes - name: - - python3-pip - - git - - - name: Link /usr/bin/pip to /usr/bin/pip3 - become: yes - command: sudo ln -s /usr/bin/pip3 /usr/bin/pip - - - name: pip install setuptools - become: yes - pip: - name: - - setuptools - - - name: clone mininet - git: - repo: https://github.com/mininet/mininet - dest: ~/mininet - version: master - - - name: install mininet - command: ~/mininet/util/install.sh -nwv - register: cmd_out - changed_when: cmd_out is not failed diff --git a/playbooks/dependencies/p4c.yml b/playbooks/dependencies/p4c.yml deleted file mode 100644 index d06c1714..00000000 --- a/playbooks/dependencies/p4c.yml +++ /dev/null @@ -1,88 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -# Requires playbook - protobuf.yml - -- hosts: all - - gather_facts: no - - tasks: - - name: apt update - become: yes - apt: - update_cache: yes - - - name: install packages for p4c - become: yes - apt: - name: - - git - - python3-pip - - g++ - - automake - - libtool - - libgc-dev - - cmake - - flex - - bison - - libfl-dev - - libgmp-dev - - libboost-dev - - libboost-iostreams-dev - - libboost-graph-dev - - llvm - - pkg-config - - tcpdump - - doxygen - - - name: pip install - become: yes - pip: - name: - - scapy - - ipaddr - - ply - - - name: clone p4c - git: - repo: https://github.com/p4lang/p4c - dest: ~/p4c - recursive: yes - version: "{{ p4c_version }}" - - - name: Create p4c build directory - file: - path: ~/p4c/build - state: directory - - - name: cmake p4c - command: cmake .. -DCMAKE_BUILD_TYPE=DEBUG -DENABLE_EBPF=OFF -DENABLE_BMV2=ON - args: - chdir: ~/p4c/build - register: cmd_out - changed_when: cmd_out is not failed - - - name: make -j4 - command: make -j4 - args: - chdir: ~/p4c/build - register: cmd_out - changed_when: cmd_out is not failed - - - name: install p4c - command: sudo make install - args: - chdir: ~/p4c/build - register: cmd_out - changed_when: cmd_out is not failed diff --git a/playbooks/dependencies/p4runtime.yml b/playbooks/dependencies/p4runtime.yml deleted file mode 100644 index 68db123c..00000000 --- a/playbooks/dependencies/p4runtime.yml +++ /dev/null @@ -1,85 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -- hosts: all - - gather_facts: no - - environment: - LDFLAGS: "-Wl,-s" - - tasks: - - name: install packages for p4runtime (PI) - become: yes - apt: - name: - - git - - autoconf - - automake - - libtool - - curl - - make - - g++ - - libjudy-dev - - pkg-config - - libboost-all-dev - - - name: clone PI - git: - repo: https://github.com/p4lang/PI - dest: ~/PI - version: "{{ pi_version }}" - recursive: yes - force: yes - - - name: autogen PI - command: ./autogen.sh - args: - chdir: ~/PI - register: cmd_out - changed_when: cmd_out is not failed - - - name: configure PI - command: ./configure --with-proto - args: - chdir: ~/PI - register: cmd_out - changed_when: cmd_out is not failed - - - name: make PI - command: make -j4 - args: - chdir: ~/PI - register: cmd_out - changed_when: cmd_out is not failed - - - name: install PI - command: sudo make install - args: - chdir: ~/PI - register: cmd_out - changed_when: cmd_out is not failed - - - name: ldconfig PI - command: sudo ldconfig - args: - chdir: ~/PI - register: cmd_out - changed_when: cmd_out is not failed - - - name: Copy Python {{ python_version }} site-packages to dist-packages PI - command: "sudo cp -r /home/ubuntu/PI/proto/py_out/{{ item }} /usr/local/lib/python{{ python_version }}/dist-packages" - with_items: - - p4 - - gnmi - - google diff --git a/playbooks/dependencies/protobuf.yml b/playbooks/dependencies/protobuf.yml deleted file mode 100644 index bfba31a4..00000000 --- a/playbooks/dependencies/protobuf.yml +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -- hosts: all - - gather_facts: no - - environment: - CFLAGS: "-Os" - CXXFLAGS: "-Os" - LDFLAGS: "-Wl,-s" - - tasks: - - name: apt update - become: yes - apt: - update_cache: yes - - - name: install packages for protobuf - become: yes - apt: - name: - - git - - autoconf - - automake - - libtool - - curl - - make - - gcc - - unzip - - - name: clone protobuf - git: - repo: https://github.com/protocolbuffers/protobuf - dest: ~/protobuf - version: "{{ protobuf_version }}" - recursive: yes - force: yes - - - name: autogen protobuf - command: ./autogen.sh - args: - chdir: ~/protobuf - register: cmd_out - changed_when: cmd_out is not failed - - - name: configure protobuf - command: ./configure --prefix=/usr - args: - chdir: ~/protobuf - register: cmd_out - changed_when: cmd_out is not failed - - - name: make protobuf - command: make -j4 - args: - chdir: ~/protobuf - register: cmd_out - changed_when: cmd_out is not failed - - - name: make install protobuf - command: sudo make install - args: - chdir: ~/protobuf - register: cmd_out - changed_when: cmd_out is not failed - - - name: ldconfig protobuf - command: sudo ldconfig - args: - chdir: ~/protobuf - register: cmd_out - changed_when: cmd_out is not failed - - - name: Install protobuf into python - shell: "cd /home/ubuntu/protobuf/python/; sudo python setup.py install" - - - name: Copy Python {{ python_version }} protobuf to dist-packages PI - command: "sudo cp -r /home/ubuntu/protobuf/python/{{ item }} /usr/local/lib/python{{ python_version }}/dist-packages" - with_items: - - google diff --git a/playbooks/env-build/ae/env_build.yml b/playbooks/env-build/ae/env_build.yml deleted file mode 100644 index f3c13b1d..00000000 --- a/playbooks/env-build/ae/env_build.yml +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# https://github.com/p4lang/tutorials/blob/master/vm/user-bootstrap.sh -# Project and script derived in part from the script in the link above ---- -- import_playbook: ../../hcp/ae_node_build.yml diff --git a/playbooks/env-build/mininet/env_build.yml b/playbooks/env-build/mininet/env_build.yml deleted file mode 100644 index aef045c1..00000000 --- a/playbooks/env-build/mininet/env_build.yml +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# https://github.com/p4lang/tutorials/blob/master/vm/user-bootstrap.sh -# Project and script derived in part from the script in the link above ---- -- import_playbook: ../../dependencies/mininet.yml -- import_playbook: ../../dependencies/protobuf.yml -- import_playbook: ../../dependencies/grpc.yml -- import_playbook: ../../dependencies/bmv2.yml -- import_playbook: ../../dependencies/p4c.yml -- import_playbook: ../../general/p4_runtime_shell.yml -- import_playbook: ../../general/final_env_setup.yml diff --git a/playbooks/general/final_env_setup.yml b/playbooks/general/final_env_setup.yml index b33e6864..1e383a52 100644 --- a/playbooks/general/final_env_setup.yml +++ b/playbooks/general/final_env_setup.yml @@ -11,8 +11,6 @@ # See the License for the specific language governing permissions and # limitations under the License. --- -# Requires playbook - protobuf.yml - - hosts: all gather_facts: no diff --git a/playbooks/general/templates/mininet_service.sh.j2 b/playbooks/general/templates/mininet_service.sh.j2 deleted file mode 100644 index 9948c4a6..00000000 --- a/playbooks/general/templates/mininet_service.sh.j2 +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash - -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Stops then starts mininet with the given topology -mn -c; {{ trans_sec_dir }}/bin/run_p4_mininet.py -t {{ topo_file_loc }} -l {{ log_dir }} diff --git a/playbooks/general/templates/sdn_controller.sh.j2 b/playbooks/general/templates/sdn_controller.sh.j2 index 9979dd5d..059fd1e2 100644 --- a/playbooks/general/templates/sdn_controller.sh.j2 +++ b/playbooks/general/templates/sdn_controller.sh.j2 @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# Runs the SDN controller for Mininet simulation +# Runs the SDN controller {{ trans_sec_dir }}/bin/sdn_controller.py -l {{ log_level }} \ -ainv {{ ansible_inventory_file }} \ -u {{ ansible_user }} \ diff --git a/playbooks/hcp/ae_node_build.yml b/playbooks/hcp/ae_node_build.yml deleted file mode 100644 index c5737b7d..00000000 --- a/playbooks/hcp/ae_node_build.yml +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -#Import playbook -- import_playbook: setup_ae_centos.yml diff --git a/playbooks/hcp/conf_ae_elk.yml b/playbooks/hcp/conf_ae_elk.yml deleted file mode 100644 index acd1ef6e..00000000 --- a/playbooks/hcp/conf_ae_elk.yml +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright (c) 2020 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -- hosts: ae - gather_facts: no - tasks: - - name: "Validate if a DNS tps.sdn.org is present in /etc/hosts file" - become: yes - tags: sdnDnsName - lineinfile: - path: /etc/hosts - regexp: "tps.sdn.org" - line: "tps.sdn.org {{ sdn_ip }}" - state: present - backup: yes - register: sdnDnsNameOut - - - name: POST call to create Index Pattern - uri: - url: "http://localhost:5601/api/saved_objects/index-pattern/packets-*" - method: POST - headers: - kbn-xsrf: "true" - Content-Type: "application/json" - return_content: yes - status_code: 200 - body: "{{ lookup('file','./templates/index_pattern_request_body.json') }}" - body_format: json - register: index_pattern_response diff --git a/playbooks/hcp/delete_ae_history.yml b/playbooks/hcp/delete_ae_history.yml deleted file mode 100644 index 31f6a4c0..00000000 --- a/playbooks/hcp/delete_ae_history.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright (c) 2020 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- hosts: ae - gather_facts: no - tasks: - - name: "Remove all packet indices" - become: yes - uri: - url: "http://localhost:9200/packets-*" - method: DELETE - headers: - kbn-xsrf: "true" - Content-Type: "application/json" - return_content: yes - status_code: 200 - register: delete_packets_index_response diff --git a/playbooks/hcp/setup_ae_centos.yml b/playbooks/hcp/setup_ae_centos.yml deleted file mode 100644 index 4ec76380..00000000 --- a/playbooks/hcp/setup_ae_centos.yml +++ /dev/null @@ -1,242 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -#Import playbook - -- hosts: all - gather_facts: no - vars: - USER_HOME: "/home/centos" - CMAKE_DIR: "{{ USER_HOME }}/cmake-3.5.2" - WIRESHARK_DIR: "{{ USER_HOME }}/wireshark-3.2.6" - PYTHON_DIR: "{{ USER_HOME }}/Python-3.8.5" - tasks: - - name: Uninstall python3-pip & python-pip - become: yes - yum: - name: - - python3-pip - - python-pip - state: absent - - - name: Install yum dependencies - become: yes - yum: - name: - - tcpdump - - vim - - wget - - unzip - - java-1.8.0-openjdk-devel - - git - - gcc - - gcc-c++ - - bison - - flex - - libpcap-devel - - qt-devel - - gtk3-devel - - rpm-build - - libtool - - c-ares-devel - - qt5-qtbase-devel - - qt5-qtmultimedia-devel - - qt5-linguist - - desktop-file-utils - - openssl-devel - - bzip2-devel - - libffi-devel - - libgcrypt-devel - - ncurses-devel - - - name: Get Wireshark v3.2.6 tar - get_url: - url: https://www.wireshark.org/download/src/all-versions/wireshark-3.2.6.tar.xz - dest: "{{ USER_HOME }}" - - - name: Create directory for Wireshark - file: - path: "{{ WIRESHARK_DIR }}" - state: directory - - - name: Extract Wireshark - unarchive: - src: "{{ USER_HOME }}/wireshark-3.2.6.tar.xz" - dest: "{{ USER_HOME }}" - remote_src: yes - - - name: Run rpm-setup script in Wireshark repository to install unmet dependencies - become: yes - command: "{{ WIRESHARK_DIR }}/tools/rpm-setup.sh -y" - - - name: Retrieve cmake tarball - get_url: - url: https://cmake.org/files/v3.5/cmake-3.5.2.tar.gz - dest: "{{ USER_HOME }}" - - - name: Create directory for cmake - file: - path: "{{ CMAKE_DIR }}" - state: directory - - - name: Extract cmake - unarchive: - src: "{{ USER_HOME }}/cmake-3.5.2.tar.gz" - dest: "{{ USER_HOME }}" - remote_src: yes - - - name: Bootstrap prefix usr before executing make for cmake - command: ./bootstrap --prefix=/usr - args: - chdir: "{{ CMAKE_DIR }}" - - - name: Run cmake make install - become: yes - command: make -j4 install - args: - chdir: "{{ CMAKE_DIR }}" - - - name: Verify cmake installed - command: cmake --version - register: cmake_out - args: - chdir: "{{ CMAKE_DIR }}" - - - name: Fail when cmake version is incorrect - fail: - msg: "cmake version is incorrect {{ cmake_out }}" - when: cmake_out.stdout is not search('3.5.2') - - - name: Create build directory for Wireshark installation - file: - path: "{{ WIRESHARK_DIR }}/build" - state: directory - - - name: Run CMAKE command for Wireshark installation - command: "cmake {{ WIRESHARK_DIR }}" - args: - chdir: "{{ WIRESHARK_DIR }}/build" - - - name: Run make Wireshark - command: make -j4 - args: - chdir: "{{ WIRESHARK_DIR }}/build" - - - name: Install Wireshark - become: yes - command: make -j4 install - args: - chdir: "{{ WIRESHARK_DIR }}/build" - - - name: Download and install a specific version of Python v3.8.5 - get_url: - url: https://www.python.org/ftp/python/3.8.5/Python-3.8.5.tgz - dest: "{{ USER_HOME }}" - - - name: Create directory for Python - file: - path: "{{ PYTHON_DIR }}" - state: directory - - - name: Extract Python-3.8.5 - unarchive: - src: "{{ USER_HOME }}/Python-3.8.5.tgz" - dest: "{{ USER_HOME }}" - remote_src: yes - - - name: Configure Python optimizations for Python 3.8 install - command: ./configure --enable-optimizations - args: - chdir: "{{ PYTHON_DIR }}" - - - name: Run INSTALL command for python 3.8 installation - become: yes - command: make -j4 altinstall - args: - chdir: "{{ PYTHON_DIR }}" - - - name: Remove previously existing python3 configuration - become: yes - file: - path: /usr/bin/python3 - state: absent - - - name: Reset Python3 version to Python3.8 installation - become: yes - shell: ln -s /usr/local/bin/python3.8 /usr/bin/python3 - - - name: Get repo OpenDistro Elasticsearch via RPM package - become: yes - get_url: - url: https://d3g5vo6xdbdb9a.cloudfront.net/yum/opendistroforelasticsearch-artifacts.repo - dest: /etc/yum.repos.d - - - name: Install OpenDistro Elasticsearch - become: yes - yum: - name: opendistroforelasticsearch-1.11.0-1 - - - name: Remove link for OpenDistro Elasticsearch installation - become: yes - file: - path: /usr/share/elasticsearch/lib/tools.jar - state: absent - - - name: Add link for OpenDistro Elasticsearch installation - become: yes - command: ln -s /usr/lib/jvm/java-1.8.0/lib/tools.jar /usr/share/elasticsearch/lib/tools.jar - - - name: Remove opendistro_security plugin for current OpenDistro Elasticsearch installation - become: yes - command: /usr/share/elasticsearch/bin/elasticsearch-plugin remove opendistro_security - - - name: Update OpenDistro Elasticsearch config file to disable security - become: yes - copy: - src: ./templates/opendistro_elasticsearch.sh - dest: /etc/elasticsearch/elasticsearch.yml - force: yes - - - name: Install OpenDistro Kibana - become: yes - yum: - name: opendistroforelasticsearch-kibana - - - name: Remove opendistro_security plugin for current OpenDistro Kibana installation - become: yes - command: /usr/share/kibana/bin/kibana-plugin remove opendistro_security --allow-root - - - name: Update OpenDistro Kibana config file - become: yes - copy: - src: ./templates/opendistro_kibana.sh - dest: /etc/kibana/kibana.yml - force: yes - - - name: Get Espcap repository - git: - repo: https://github.com/vichargrave/espcap - dest: "{{ USER_HOME }}/espcap" - - - name: Install requirements for Espcap Elasticsearch v7.x - become: yes - command: "pip3 install -r {{ USER_HOME }}/espcap/requirements-7.x.txt" - - - name: Copy TPS requirements.txt - copy: - src: ../../requirements.txt - dest: "{{ USER_HOME }}/tps-py-requirements.txt" - - - name: Install TPS Python requirements - become: yes - command: "pip3 install -r {{ USER_HOME }}/tps-py-requirements.txt" diff --git a/playbooks/hcp/setup_ae_tcp_pipeline.yml b/playbooks/hcp/setup_ae_tcp_pipeline.yml deleted file mode 100644 index c1f285da..00000000 --- a/playbooks/hcp/setup_ae_tcp_pipeline.yml +++ /dev/null @@ -1,165 +0,0 @@ -# Copyright (c) 2020 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#Import playbook -- import_playbook: setup_ae_elk.yml - -- hosts: ae - gather_facts: no - tasks: - - name: POST call to create TCP data mapping template for incoming data in Elasticsearch - uri: - url: "http://localhost:9200/_template/packets" - method: PUT - return_content: yes - body: "{{ lookup('file','./templates/tcp/tcp_data_mapping.json') }}" - body_format: json - register: tcp_template_response - - - name: POST call to create data parsing pipeline for TCP v4 and UDP v6 for incoming data in Elasticsearch - uri: - url: "http://localhost:9200/_ingest/pipeline/ts_tcp_parsing" - method: PUT - return_content: yes - body: "{{ lookup('file','./templates/tcp/tcp_data_parsing.json') }}" - body_format: json - register: ts_pipeline_response - - - name: Ensures {{ remote_scripts_dir }} exists - become: yes - file: - path: "{{ remote_scripts_dir }}" - state: directory - - - name: Ensure {{ remote_scripts_dir }} directories are 777 - become: yes - shell: "chmod -R 777 transparent-security" - args: - chdir: "/etc" - - - name: Copy file sdn_attack_webhook.json to {{ remote_scripts_dir }} for updates - copy: - src: "{{ trans_sec_dir }}/playbooks/hcp/templates/sdn_attack_webhook.json" - dest: "{{ remote_scripts_dir }}/sdn_attack_webhook.json" - remote_src: yes - force: yes - - - name: Replace SDN Attack Webhook definition file for SDN port - become: yes - tags: sdnWebhookUpdateForPort - lineinfile: - path: "{{ remote_scripts_dir }}/sdn_attack_webhook.json" - regex: 'sdn_port' - line: '"port": "{{sdn_port}}",' - state: present - backup: yes - register: sdnWebhookUpdateForPort - - - name: Replace SDN Attack Webhook definition file for SDN IP - become: yes - tags: sdnWebhookUpdateForIP - lineinfile: - path: "{{ remote_scripts_dir }}/sdn_attack_webhook.json" - regex: 'sdn_ip' - line: '"host": "{{sdn_ip}}"' - state: present - backup: yes - register: sdnWebhookUpdateForIP - - - name: Slurp updated sdn webhook definition and save it to be used in POST request. - slurp: - src: "{{ remote_scripts_dir }}/sdn_attack_webhook.json" - register: slurped_updated_sdn_webhook_def - - - name: Set updated_sdn_webhook_def as fact - set_fact: - updated_sdn_webhook_def_contents: "{{ slurped_updated_sdn_webhook_def.content | b64decode }}" - - - name: Show SDN Webhook definition - debug: - var: updated_sdn_webhook_def_contents - - - - name: POST call to create SDN Aggregate attack webhook to attach to monitor for incoming data in Elasticsearch - uri: - url: "http://localhost:9200/_opendistro/_alerting/destinations" - method: POST - return_content: yes - status_code: 201 - body: "{{ updated_sdn_webhook_def_contents }}" - body_format: json - register: sdn_webhook_response - - - name: Set SDN Webhook call response - set_fact: - sdn_webhook_response: "{{ sdn_webhook_response.json }}" - - - name: Set SDN Webhook ID from response - set_fact: - sdn_webhook_id: "{{ sdn_webhook_response._id }}" - - - name: Fetch created SDN webhook ID for creating associated monitor - debug: - msg: " SDN Webhook response Id : {{ sdn_webhook_id }}" - - - name: Ensures {{ remote_scripts_dir }} exists - become: yes - file: - path: "{{ remote_scripts_dir }}" - state: directory - - - name: Ensure {{ remote_scripts_dir }} directories are 777 - become: yes - shell: "chmod -R 777 transparent-security" - args: - chdir: "/etc" - - - name: Copy file udp_monitor_def.json to {{ remote_scripts_dir }} for updates - copy: - src: "{{ trans_sec_dir }}/playbooks/hcp/templates/tcp/tcp_monitor_def.json" - dest: "{{ remote_scripts_dir }}/tcp_monitor_def.json" - remote_src: yes - force: yes - - - name: Replace SDN Webhook ID in TCP monitor definition file - become: yes - tags: monitorUpdate - lineinfile: - path: "{{ remote_scripts_dir }}/tcp_monitor_def.json" - regex: 'sdnWebhookResponseId' - line: ' "destination_id": "{{sdn_webhook_id}}",' - state: present - backup: yes - register: monitorUpdateOut - - - name: Slurp updated monitor definition and save it to be used in POST request. - slurp: - src: "{{ remote_scripts_dir }}/tcp_monitor_def.json" - register: slurped_updated_tcp_monitor_def - - - name: Set updated_tcp_monitor_def as fact - set_fact: - updated_tcp_monitor_def_contents: "{{ slurped_updated_tcp_monitor_def.content | b64decode }}" - - - name: Show Updated monitor definition - debug: - var: updated_tcp_monitor_def_contents - - - name: POST call to create Monitor,trigger and action for TCP DDOS Detection and Mitigation Pipeline for incoming data in Elasticsearch - uri: - url: "http://localhost:9200/_opendistro/_alerting/monitors" - method: POST - return_content: yes - status_code: 201 - body: "{{ updated_tcp_monitor_def_contents }}" - body_format: json - register: tcp_monitor_response diff --git a/playbooks/hcp/setup_ae_udp_and_tcp_pipeline.yml b/playbooks/hcp/setup_ae_udp_and_tcp_pipeline.yml deleted file mode 100644 index 3d5bf12a..00000000 --- a/playbooks/hcp/setup_ae_udp_and_tcp_pipeline.yml +++ /dev/null @@ -1,195 +0,0 @@ -# Copyright (c) 2020 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#Import playbook -- import_playbook: conf_ae_elk.yml - -- hosts: ae - gather_facts: no - tasks: - - name: POST call to create data mapping template for incoming UDP and TCP data in Elasticsearch - uri: - url: "http://localhost:9200/_template/packets" - method: PUT - return_content: yes - body: "{{ lookup('file','./templates/udp_and_tcp_data_mapping.json') }}" - body_format: json - register: udp_template_response - - - name: POST call to create script function to generate drop report hash - uri: - url: "http://localhost:9200/_scripts/calculateKeyHash" - method: POST - return_content: yes - body: "{{ lookup('file','./templates/drop_report_hash_script.json') }}" - body_format: json - register: drop_report_hash_response - - - name: POST call to create data parsing pipeline for UDP and TCP v4 and UDP v6 for incoming data in Elasticsearch - uri: - url: "http://localhost:9200/_ingest/pipeline/ts_parsing" - method: PUT - return_content: yes - body: "{{ lookup('file','./templates/udp_and_tcp_data_parsing.json') }}" - body_format: json - register: ts_pipeline_response - - - name: Ensures {{ remote_scripts_dir }} exists - become: yes - file: - path: "{{ remote_scripts_dir }}" - state: directory - - - name: Ensure {{ remote_scripts_dir }} directories are 777 - become: yes - shell: "chmod -R 777 transparent-security" - args: - chdir: "/etc" - - - name: Copy file sdn_attack_webhook.json to {{ remote_scripts_dir }} for updates - copy: - src: "{{ trans_sec_dir }}/playbooks/ae/templates/sdn_attack_webhook.json" - dest: "{{ remote_scripts_dir }}/sdn_attack_webhook.json" - remote_src: yes - force: yes - - - name: Replace SDN Attack Webhook definition file for SDN port - become: yes - tags: sdnWebhookUpdateForPort - lineinfile: - path: "{{ remote_scripts_dir }}/sdn_attack_webhook.json" - regex: 'sdn_port' - line: '"port": "{{sdn_port}}",' - state: present - backup: yes - register: sdnWebhookUpdateForPort - - - name: Replace SDN Attack Webhook definition file for SDN IP - become: yes - tags: sdnWebhookUpdateForIP - lineinfile: - path: "{{ remote_scripts_dir }}/sdn_attack_webhook.json" - regex: 'sdn_ip' - line: '"host": "{{sdn_ip}}"' - state: present - backup: yes - register: sdnWebhookUpdateForIP - - - name: Slurp updated sdn webhook definition and save it to be used in POST request. - slurp: - src: "{{ remote_scripts_dir }}/sdn_attack_webhook.json" - register: slurped_updated_sdn_webhook_def - - - name: Set updated_sdn_webhook_def as fact - set_fact: - updated_sdn_webhook_def_contents: "{{ slurped_updated_sdn_webhook_def.content | b64decode }}" - - - name: Show SDN Webhook definition - debug: - var: updated_sdn_webhook_def_contents - - - name: POST call to create SDN Aggregate attack webhook to attach to monitor for incoming data in Elasticsearch - uri: - url: "http://localhost:9200/_opendistro/_alerting/destinations" - method: POST - return_content: yes - status_code: 201 - body: "{{ updated_sdn_webhook_def_contents }}" - body_format: json - register: sdn_webhook_response - - - name: Set SDN Webhook call response - set_fact: - sdn_webhook_response: "{{ sdn_webhook_response.json }}" - - - name: Set SDN Webhook ID from response - set_fact: - sdn_webhook_id: "{{ sdn_webhook_response._id }}" - - - name: Fetch created SDN webhook ID for creating associated monitor - debug: - msg: " SDN Webhook response Id : {{ sdn_webhook_id }}" - - - - name: Copy file udp_monitor_def.json to {{ remote_scripts_dir }} for updates - copy: - src: "{{ trans_sec_dir }}/playbooks/ae/templates/udp_and_tcp_monitor_def.json" - dest: "{{ remote_scripts_dir }}/udp_and_tcp_monitor_def.json" - remote_src: yes - force: yes - - - name: Replace SDN Webhook ID in UDP monitor definition file for TCPv4 trigger - become: yes - tags: monitorUpdate - lineinfile: - path: "{{ remote_scripts_dir }}/udp_and_tcp_monitor_def.json" - regex: 'sdnWebhookResponseIdTCPv4' - line: ' "destination_id": "{{sdn_webhook_id}}",' - state: present - backup: yes - register: monitorUpdateOut - - - name: Replace SDN Webhook ID in UDP monitor definition file for TCPv6 trigger - become: yes - tags: monitorUpdate - lineinfile: - path: "{{ remote_scripts_dir }}/udp_and_tcp_monitor_def.json" - regex: 'sdnWebhookResponseIdTCPv6' - line: ' "destination_id": "{{sdn_webhook_id}}",' - state: present - backup: yes - register: monitorUpdateOut - - - name: Replace SDN Webhook ID in UDP monitor definition file for UDPv4 trigger - become: yes - tags: monitorUpdate - lineinfile: - path: "{{ remote_scripts_dir }}/udp_and_tcp_monitor_def.json" - regex: 'sdnWebhookResponseIdUDPv4' - line: ' "destination_id": "{{sdn_webhook_id}}",' - state: present - backup: yes - register: monitorUpdateOut - - - name: Replace SDN Webhook ID in UDP monitor definition file for UDPv6 trigger - become: yes - tags: monitorUpdate - lineinfile: - path: "{{ remote_scripts_dir }}/udp_and_tcp_monitor_def.json" - regex: 'sdnWebhookResponseIdUDPv6' - line: ' "destination_id": "{{sdn_webhook_id}}",' - state: present - backup: yes - register: monitorUpdateOut - - - name: Slurp updated monitor definition and save it to be used in POST request. - slurp: - src: "{{ remote_scripts_dir }}/udp_and_tcp_monitor_def.json" - register: slurped_updated_udp_and_tcp_monitor_def - - - name: Set updated_udp_monitor_def as fact - set_fact: - updated_udp_and_tcp_monitor_def_contents: "{{ slurped_updated_udp_and_tcp_monitor_def.content | b64decode }}" - - - name: Show Updated monitor definition for UDP and TCP pipeline - debug: - var: updated_udp_and_tcp_monitor_def_contents - - - name: POST call to create Monitor,trigger and action for UDP DDOS Detection and Mitigation Pipeline for incoming data in Elasticsearch - uri: - url: "http://localhost:9200/_opendistro/_alerting/monitors" - method: POST - return_content: yes - status_code: 201 - body: "{{ updated_udp_and_tcp_monitor_def_contents }}" - body_format: json - register: udp_and_tcp_monitor_response diff --git a/playbooks/hcp/setup_ae_udp_pipeline.yml b/playbooks/hcp/setup_ae_udp_pipeline.yml deleted file mode 100644 index e194daed..00000000 --- a/playbooks/hcp/setup_ae_udp_pipeline.yml +++ /dev/null @@ -1,162 +0,0 @@ -# Copyright (c) 2020 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#Import playbook -- import_playbook: conf_ae_elk.yml - -- hosts: ae - gather_facts: no - tasks: - - name: POST call to create data mapping template for incoming data in Elasticsearch - uri: - url: "http://localhost:9200/_template/packets" - method: PUT - return_content: yes - body: "{{ lookup('file','./templates/udp/udp_data_mapping.json') }}" - body_format: json - register: udp_template_response - - - name: POST call to create script function to generate drop report hash - uri: - url: "http://localhost:9200/_scripts/calculateKeyHash" - method: POST - return_content: yes - body: "{{ lookup('file','./templates/drop_report_hash_script.json') }}" - body_format: json - register: drop_report_hash_response - - - name: POST call to create data parsing pipeline for UDP v4 and UDP v6 for incoming data in Elasticsearch - uri: - url: "http://localhost:9200/_ingest/pipeline/ts_parsing" - method: PUT - return_content: yes - body: "{{ lookup('file','./templates/udp/udp_data_parsing.json') }}" - body_format: json - register: ts_pipeline_response - - - name: Ensures {{ remote_scripts_dir }} exists - become: yes - file: - path: "{{ remote_scripts_dir }}" - state: directory - - - name: Ensure {{ remote_scripts_dir }} directories are 777 - become: yes - shell: "chmod -R 777 transparent-security" - args: - chdir: "/etc" - - - name: Copy file sdn_attack_webhook.json to {{ remote_scripts_dir }} for updates - copy: - src: "{{ trans_sec_dir }}/playbooks/hcp/templates/sdn_attack_webhook.json" - dest: "{{ remote_scripts_dir }}/sdn_attack_webhook.json" - remote_src: yes - force: yes - - - name: Replace SDN Attack Webhook definition file for SDN port - become: yes - tags: sdnWebhookUpdateForPort - lineinfile: - path: "{{ remote_scripts_dir }}/sdn_attack_webhook.json" - regex: 'sdn_port' - line: '"port": "{{sdn_port}}",' - state: present - backup: yes - register: sdnWebhookUpdateForPort - - - name: Replace SDN Attack Webhook definition file for SDN IP - become: yes - tags: sdnWebhookUpdateForIP - lineinfile: - path: "{{ remote_scripts_dir }}/sdn_attack_webhook.json" - regex: 'sdn_ip' - line: '"host": "{{sdn_ip}}"' - state: present - backup: yes - register: sdnWebhookUpdateForIP - - - name: Slurp updated sdn webhook definition and save it to be used in POST request. - slurp: - src: "{{ remote_scripts_dir }}/sdn_attack_webhook.json" - register: slurped_updated_sdn_webhook_def - - - name: Set updated_sdn_webhook_def as fact - set_fact: - updated_sdn_webhook_def_contents: "{{ slurped_updated_sdn_webhook_def.content | b64decode }}" - - - name: Show SDN Webhook definition - debug: - var: updated_sdn_webhook_def_contents - - - name: POST call to create SDN Aggregate attack webhook to attach to monitor for incoming data in Elasticsearch - uri: - url: "http://localhost:9200/_opendistro/_alerting/destinations" - method: POST - return_content: yes - status_code: 201 - body: "{{ updated_sdn_webhook_def_contents }}" - body_format: json - register: sdn_webhook_response - - - name: Set SDN Webhook call response - set_fact: - sdn_webhook_response: "{{ sdn_webhook_response.json }}" - - - name: Set SDN Webhook ID from response - set_fact: - sdn_webhook_id: "{{ sdn_webhook_response._id }}" - - - name: Fetch created SDN webhook ID for creating associated monitor - debug: - msg: " SDN Webhook response Id : {{ sdn_webhook_id }}" - - - - name: Copy file udp_monitor_def.json to {{ remote_scripts_dir }} for updates - copy: - src: "{{ trans_sec_dir }}/playbooks/hcp/templates/udp/udp_monitor_def.json" - dest: "{{ remote_scripts_dir }}/udp_monitor_def.json" - remote_src: yes - force: yes - - - name: Replace SDN Webhook ID in UDP monitor definition file - become: yes - tags: monitorUpdate - lineinfile: - path: "{{ remote_scripts_dir }}/udp_monitor_def.json" - regex: 'sdnWebhookResponseId' - line: ' "destination_id": "{{sdn_webhook_id}}",' - state: present - backup: yes - register: monitorUpdateOut - - - name: Slurp updated monitor definition and save it to be used in POST request. - slurp: - src: "{{ remote_scripts_dir }}/udp_monitor_def.json" - register: slurped_updated_udp_monitor_def - - - name: Set updated_udp_monitor_def as fact - set_fact: - updated_udp_monitor_def_contents: "{{ slurped_updated_udp_monitor_def.content | b64decode }}" - - - name: Show Updated monitor definition - debug: - var: updated_udp_monitor_def_contents - - - name: POST call to create Monitor,trigger and action for UDP DDOS Detection and Mitigation Pipeline for incoming data in Elasticsearch - uri: - url: "http://localhost:9200/_opendistro/_alerting/monitors" - method: POST - return_content: yes - status_code: 201 - body: "{{ updated_udp_monitor_def_contents }}" - body_format: json - register: udp_monitor_response diff --git a/playbooks/hcp/start_ae_elk.yml b/playbooks/hcp/start_ae_elk.yml deleted file mode 100644 index 0461bee5..00000000 --- a/playbooks/hcp/start_ae_elk.yml +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright (c) 2020 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -- hosts: ae - gather_facts: no - vars: - kibana_wait: "{{ wait_for_kibana | default(180) }}" - tasks: - - name: "Start Elasticsearch service" - become: yes - service: - name: elasticsearch - state: started - enabled: yes - - - name: "Start Kibana service" - become: yes - service: - name: kibana - state: started - enabled: yes - - - name: Wait for Kibana to startup at port 5601 - wait_for: - port: 5601 - timeout: 600 - - - name: Wait for kibana - pause: - seconds: "{{ kibana_wait }}" diff --git a/playbooks/hcp/templates/drop_report_hash_script.json b/playbooks/hcp/templates/drop_report_hash_script.json deleted file mode 100644 index 457b5bfd..00000000 --- a/playbooks/hcp/templates/drop_report_hash_script.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "script" : { - "lang" : "painless", - "source" : """ -int[] HS() { - return new int[] { - (int) 0x6a09e667L, - (int) 0xbb67ae85L, - (int) 0x3c6ef372L, - (int) 0xa54ff53aL, - (int) 0x510e527fL, - (int) 0x9b05688cL, - (int) 0x1f83d9abL, - (int) 0x5be0cd19L - }; -} -int[] KS() { -return new int[] { - (int) 0x428a2f98L, (int) 0x71374491L, (int) 0xb5c0fbcfL, (int) 0xe9b5dba5L, - (int) 0x3956c25bL, (int) 0x59f111f1L, (int) 0x923f82a4L, (int) 0xab1c5ed5L, - (int) 0xd807aa98L, (int) 0x12835b01L, (int) 0x243185beL, (int) 0x550c7dc3L, - (int) 0x72be5d74L, (int) 0x80deb1feL, (int) 0x9bdc06a7L, (int) 0xc19bf174L, - (int) 0xe49b69c1L, (int) 0xefbe4786L, (int) 0x0fc19dc6L, (int) 0x240ca1ccL, - (int) 0x2de92c6fL, (int) 0x4a7484aaL, (int) 0x5cb0a9dcL, (int) 0x76f988daL, - (int) 0x983e5152L, (int) 0xa831c66dL, (int) 0xb00327c8L, (int) 0xbf597fc7L, - (int) 0xc6e00bf3L, (int) 0xd5a79147L, (int) 0x06ca6351L, (int) 0x14292967L, - (int) 0x27b70a85L, (int) 0x2e1b2138L, (int) 0x4d2c6dfcL, (int) 0x53380d13L, - (int) 0x650a7354L, (int) 0x766a0abbL, (int) 0x81c2c92eL, (int) 0x92722c85L, - (int) 0xa2bfe8a1L, (int) 0xa81a664bL, (int) 0xc24b8b70L, (int) 0xc76c51a3L, - (int) 0xd192e819L, (int) 0xd6990624L, (int) 0xf40e3585L, (int) 0x106aa070L, - (int) 0x19a4c116L, (int) 0x1e376c08L, (int) 0x2748774cL, (int) 0x34b0bcb5L, - (int) 0x391c0cb3L, (int) 0x4ed8aa4aL, (int) 0x5b9cca4fL, (int) 0x682e6ff3L, - (int) 0x748f82eeL, (int) 0x78a5636fL, (int) 0x84c87814L, (int) 0x8cc70208L, - (int) 0x90befffaL, (int) 0xa4506cebL, (int) 0xbef9a3f7L, (int) 0xc67178f2L - }; -} -byte[] digest(byte[] data, byte[] block, int[] words) { - byte[] padded = padMessage(data); - int[] hs = HS(); - - for (int i = 0; i < padded.length / 64; ++i) { - int[] registers = new int[8]; - for (int k = 0; k < 8; ++k) - registers[k] = hs[k]; - - System.arraycopy(padded, 64 * i, block, 0, 64); - - setupWords(block, words); - int[] ks = KS(); - - for (int j = 0; j < 64; ++j) { - iterate(registers, words, j, ks); - } - - for (int j = 0; j < 8; ++j) { - hs[j] += registers[j]; - } - } - byte[] hash = new byte[32]; - for (int i = 0; i < 8; i++) { - System.arraycopy(intToBytes(hs[i]), 0, hash, 4 * i, 4); - } - return hash; -} -void setupWords(byte[] block, int[] words) { - for (int j = 0; j < 16; j++) { - words[j] = 0; - for (int m = 0; m < 4; m++) { - words[j] |= ((block[j * 4 + m] & 255) << (24 - m * 8)); - } - } - for (int j = 16; j < 64; ++j) { - int s0 = Integer.rotateRight(words[j - 15], 7) ^ - Integer.rotateRight(words[j - 15], 18) ^ - (words[j - 15] >>> 3); - int s1 = Integer.rotateRight(words[j - 2], 17) ^ - Integer.rotateRight(words[j - 2], 19) ^ - (words[j - 2] >>> 10); - words[j] = words[j - 16] + s0 + words[j - 7] + s1; - } -} -void iterate(int[] registers, int[] words, int j, int[] ks) { - int S0 = Integer.rotateRight(registers[0], 2) ^ - Integer.rotateRight(registers[0], 13) ^ - Integer.rotateRight(registers[0], 22); - int maj = (registers[0] & registers[1]) ^ (registers[0] & registers[2]) ^ (registers[1] & registers[2]); - int temp2 = S0 + maj; - int S1 = Integer.rotateRight(registers[4], 6) ^ - Integer.rotateRight(registers[4], 11) ^ - Integer.rotateRight(registers[4], 25); - int ch = (registers[4] & registers[5]) ^ (~registers[4] & registers[6]); - int temp1 = registers[7] + S1 + ch + ks[j] + words[j]; - registers[7] = registers[6]; - registers[6] = registers[5]; - registers[5] = registers[4]; - registers[4] = registers[3] + temp1; - registers[3] = registers[2]; - registers[2] = registers[1]; - registers[1] = registers[0]; - registers[0] = temp1 + temp2; -} -byte[] padMessage(byte[] data) { - int length = data.length; - int tail = length % 64; - int padding; - if ((64 - tail >= 9)) { - padding = 64 - tail; - } else { - padding = 128 - tail; - } - byte[] pad = new byte[padding]; - pad[0] = (byte) 128; - long bits = length * 8; - for (int i = 0; i < 8; i++) { - pad[pad.length - 1 - i] = (byte) ((bits >>> (8 * i)) & 255); - } - byte[] output = new byte[length + padding]; - System.arraycopy(data, 0, output, 0, length); - System.arraycopy(pad, 0, output, length, pad.length); - return output; -} -byte[] intToBytes(int i) { - byte[] b = new byte[4]; - for (int c = 0; c < 4; c++) { - b[c] = (byte) ((i >>> (56 - 8 * c)) & 255); - } - return b; -} -String bytesToHex(byte[] bytes) { - char[] hexArray = "0123456789abcdef".toCharArray(); - char[] hexChars = new char[bytes.length * 2]; - for ( int j = 0; j < bytes.length; j++ ) { - int v = bytes[j] & 255; - hexChars[j * 2] = hexArray[v >>> 4]; - hexChars[j * 2 + 1] = hexArray[v & 15]; - } - return String.copyValueOf(hexChars); -} -byte[] charArrayToByteArray(char[] charArray) { -byte[] byteArray = new byte[charArray.length]; -for(int i= 0; i < charArray.length; i++) { - byteArray[i] = (byte)(255 & (int)charArray[i]); - } -return byteArray; -} -byte[] block = new byte[64]; -int[] words = new int[64]; -String dropHashInput = ctx.ts.dropHashInput; -if(dropHashInput != null && ctx.ts.isDropReport == false) -{ -byte[] data = charArrayToByteArray(dropHashInput.toCharArray()); -byte[] res = digest(data, block, words); -String sha256Str = bytesToHex(res); -ctx.ts.sha256Hash = sha256Str; -} - """ - } -} diff --git a/playbooks/hcp/templates/index_pattern_request_body.json b/playbooks/hcp/templates/index_pattern_request_body.json deleted file mode 100644 index 135f81a2..00000000 --- a/playbooks/hcp/templates/index_pattern_request_body.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "attributes": { - "title": "packets-*", - "timeFieldName": "timestamp" - } -} diff --git a/playbooks/hcp/templates/opendistro_elasticsearch.sh b/playbooks/hcp/templates/opendistro_elasticsearch.sh deleted file mode 100644 index f52c022a..00000000 --- a/playbooks/hcp/templates/opendistro_elasticsearch.sh +++ /dev/null @@ -1,113 +0,0 @@ -# ======================== Elasticsearch Configuration ========================= -# -# NOTE: Elasticsearch comes with reasonable defaults for most settings. -# Before you set out to tweak and tune the configuration, make sure you -# understand what are you trying to accomplish and the consequences. -# -# The primary way of configuring a node is via this file. This template lists -# the most important settings you may want to configure for a production cluster. -# -# Please consult the documentation for further information on configuration options: -# https://www.elastic.co/guide/en/elasticsearch/reference/index.html -# -# ---------------------------------- Cluster ----------------------------------- -# -# Use a descriptive name for your cluster: -# -cluster.name: ae-elasticsearch -# -# ------------------------------------ Node ------------------------------------ -# -# Use a descriptive name for the node: -# -#node.name: node-1 -# -# Add custom attributes to the node: -# -#node.attr.rack: r1 -# -# ----------------------------------- Paths ------------------------------------ -# -# Path to directory where to store the data (separate multiple locations by comma): -# -path.data: /var/lib/elasticsearch -# -# Path to log files: -# -path.logs: /var/log/elasticsearch -# -# ----------------------------------- Memory ----------------------------------- -# -# Lock the memory on startup: -# -#bootstrap.memory_lock: true -# -# Make sure that the heap size is set to about half the memory available -# on the system and that the owner of the process is allowed to use this -# limit. -# -# Elasticsearch performs poorly when the system is swapping the memory. -# -# ---------------------------------- Network ----------------------------------- -# -# Set the bind address to a specific IP (IPv4 or IPv6): -# -#network.host: 0.0.0.0 -# -# Set a custom port for HTTP: -# -#http.port: 9200 -# -# For more information, consult the network module documentation. -# -# --------------------------------- Discovery ---------------------------------- -# -# Pass an initial list of hosts to perform discovery when this node is started: -# The default list of hosts is ["127.0.0.1", "[::1]"] -# -#discovery.seed_hosts: ["127.0.0.1"] -# -# Bootstrap the cluster using an initial set of master-eligible nodes: -# -#cluster.initial_master_nodes: ["node-1", "node-2"] -# -# For more information, consult the discovery and cluster formation module documentation. -# -# ---------------------------------- Gateway ----------------------------------- -# -# Block initial recovery after a full cluster restart until N nodes are started: -# -#gateway.recover_after_nodes: 3 -# -# For more information, consult the gateway module documentation. -# -# ---------------------------------- Various ----------------------------------- -# -# Require explicit names when deleting indices: -# -#action.destructive_requires_name: true -######## Start OpenDistro for Elasticsearch Security Demo Configuration ######## -# WARNING: revise all the lines below before you go into production -#opendistro_security.disabled: true -#opendistro_security.ssl.transport.pemcert_filepath: esnode.pem -#opendistro_security.ssl.transport.pemkey_filepath: esnode-key.pem -#opendistro_security.ssl.transport.pemtrustedcas_filepath: root-ca.pem -#opendistro_security.ssl.transport.enforce_hostname_verification: false -#opendistro_security.ssl.http.enabled: true -#opendistro_security.ssl.http.pemcert_filepath: esnode.pem -#opendistro_security.ssl.http.pemkey_filepath: esnode-key.pem -#opendistro_security.ssl.http.pemtrustedcas_filepath: root-ca.pem -#opendistro_security.allow_unsafe_democertificates: true -#opendistro_security.allow_default_init_securityindex: true -#opendistro_security.authcz.admin_dn: -# - CN=kirk,OU=client,O=client,L=test, C=de -#opendistro_security.audit.type: internal_elasticsearch -#opendistro_security.enable_snapshot_restore_privilege: true -#opendistro_security.check_snapshot_restore_write_privileges: true -#opendistro_security.restapi.roles_enabled: ["all_access", "security_rest_api_access"] -#opendistro_security.system_indices.enabled: true -#opendistro_security.system_indices.indices: [".opendistro-alerting-config", ".opendistro-alerting-alert*"] -cluster.routing.allocation.disk.threshold_enabled: false -node.max_local_storage_nodes: 3 -######## End OpenDistro for Elasticsearch Security Demo Configuration ######## - diff --git a/playbooks/hcp/templates/opendistro_kibana.sh b/playbooks/hcp/templates/opendistro_kibana.sh deleted file mode 100644 index 1052065f..00000000 --- a/playbooks/hcp/templates/opendistro_kibana.sh +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). -# You may not use this file except in compliance with the License. -# A copy of the License is located at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# or in the "license" file accompanying this file. This file is distributed -# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either -# express or implied. See the License for the specific language governing -# permissions and limitations under the License. -# Description: -# Default Kibana configuration for Open Distro. - -server.port: 5601 -server.host: 0.0.0.0 -elasticsearch.hosts: http://localhost:9200 -elasticsearch.ssl.verificationMode: none -#elasticsearch.username: admin -#elasticsearch.password: admin -#elasticsearch.requestHeadersWhitelist: ["securitytenant","Authorization"] -#opendistro_security.multitenancy.enabled: true -#opendistro_security.multitenancy.tenants.preferred: ["Private", "Global"] -#opendistro_security.readonly_mode.roles: ["kibana_read_only"] -# Use this setting if you are running kibana without https -#opendistro_security.cookie.secure: false -newsfeed.enabled: false -telemetry.optIn: false -telemetry.enabled: false - diff --git a/playbooks/hcp/templates/query_monitor_id.json b/playbooks/hcp/templates/query_monitor_id.json deleted file mode 100644 index 1a9f2041..00000000 --- a/playbooks/hcp/templates/query_monitor_id.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "query": { - "match" : { - "monitor.name": "DDOS-UDP-TCP-Monitor" - } - } -} diff --git a/playbooks/hcp/templates/sdn_attack_webhook.json b/playbooks/hcp/templates/sdn_attack_webhook.json deleted file mode 100644 index 9fded291..00000000 --- a/playbooks/hcp/templates/sdn_attack_webhook.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "type": "custom_webhook", - "name": "SDN_Webhook", - "custom_webhook": { - "path": "aggAttack", - "header_params": { - "Content-Type": "application/json" - }, - "scheme": "HTTP", - "port": "sdn_port", - "host": "sdn_ip" - } -} diff --git a/playbooks/hcp/templates/tcp/tcp_data_mapping.json b/playbooks/hcp/templates/tcp/tcp_data_mapping.json deleted file mode 100644 index 80808a29..00000000 --- a/playbooks/hcp/templates/tcp/tcp_data_mapping.json +++ /dev/null @@ -1,292 +0,0 @@ -{ - "index_patterns" : [ - "packets-*" - ], - "settings" : { - "index" : { - "number_of_replicas" : "0", - "default_pipeline" : "ts_tcp_parsing", - "max_script_fields" : "200" - } - }, - "mappings" : { - "dynamic" : "false", - "properties" : { - "layers" : { - "properties" : { - "data_raw" : { - "type" : "keyword" - }, - "data" : { - "properties" : { - "data_data_data_raw" : { - "type" : "keyword" - }, - "data_data_len" : { - "type" : "keyword" - }, - "data_data_data" : { - "type" : "keyword" - } - } - } - } - }, - "timestamp" : { - "type" : "date" - }, - "ts" : { - "properties" : { - "INTMetadataSourceMetadataReserved" : { - "type" : "keyword" - }, - - "UDPIntShimHeaderType" : { - "type" : "keyword" - }, - "telemetryReportLength" : { - "type" : "keyword" - }, - "data" : { - "type" : "keyword" - }, - "ethernetHeaderDestMac" : { - "type" : "keyword" - }, - "telemetryRsvd" : { - "type" : "keyword" - }, - "UDGSrcPort" : { - "type" : "keyword" - }, - "INTMetadataSourceMetadataSwitchId" : { - "type" : "keyword" - }, - "ethernetHeaderSrcMac" : { - "type" : "keyword" - }, - "UDPIntShimHeaderNextProto" : { - "type" : "keyword" - }, - "isIpv4" : { - "type" : "keyword" - }, - "nextProto": { - "type" : "keyword" - }, - "isTCP" : { - "type" : "keyword" - }, - "isUDP" : { - "type" : "keyword" - }, - "TCPData" : { - "type" : "keyword" - }, - "TCPSrcPort" : { - "type" : "keyword" - }, - "TCPDestPort" : { - "type" : "keyword" - }, - "TCPIntShimHeaderNextProto" : { - "type" : "keyword" - }, - - "UDPIntMetaDataHeaderVersion" : { - "type" : "keyword" - }, - "IPvNextHdrProto" : { - "type" : "keyword" - }, - "telemetryReportVersion" : { - "type" : "keyword" - }, - "isIpv6" : { - "type" : "keyword" - }, - "IPv4SrcIP": { - "type" : "keyword" - }, - "IPv6SrcIP": { - "type" : "keyword" - }, - "IPv4DestIP":{ - "type" : "keyword" - }, - "IPv6DestIP":{ - "type" : "keyword" - }, - "INTMetadataSourceMetadataOriginatingMac" : { - "type" : "keyword" - }, - - "UDPIntMetaDataHeaderInstructionBitmap" : { - "type" : "keyword" - }, - "INTMetadataSourceMetadata" : { - "type" : "keyword" - }, - "IPvSrcAddr" : { - "type" : "keyword" - }, - "tsRawDataSize" : { - "type" : "keyword" - }, - "UDGPData" : { - "type" : "keyword" - }, - "UDGChecksum" : { - "type" : "keyword" - }, - "INTMetadataHeaderData" : { - "type" : "keyword" - }, - "IPvDestAddr" : { - "type" : "keyword" - }, - "telemetryInType" : { - "type" : "keyword" - }, - "telemetryReport" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderDSFlags" : { - "type" : "keyword" - }, - "telemetryDsMdStatus" : { - "type" : "keyword" - }, - "InBandNetworkTelemetryData" : { - "type" : "keyword" - }, - "UDPIntShimHeaderRes2" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderDSInstruction" : { - "type" : "keyword" - }, - "UDPIntShimHeaderRes1" : { - "type" : "keyword" - }, - "UDP2Data" : { - "type" : "keyword" - }, - "telemetryHwdId" : { - "type" : "keyword" - }, - - "telemetryVarOpt" : { - "type" : "keyword" - }, - "IPvData" : { - "type" : "keyword" - }, - "ethernetHeaderEtherType" : { - "type" : "keyword" - }, - "INTMetadataStackData" : { - "type" : "keyword" - }, - "IPvFlow" : { - "type" : "keyword" - }, - "UDPIntShimHeaderData" : { - "type" : "keyword" - }, - "INTMetadataStackSwitchID" : { - "type" : "keyword" - }, - "IPVersion" : { - "type" : "keyword" - }, - "IPvPayloadLength" : { - "type" : "keyword" - }, - "UDPIntShimHeaderLength" : { - "type" : "keyword" - }, - "UDPIntShimHeaderNPT" : { - "type" : "keyword" - }, - "UDGLength" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderM" : { - "type" : "keyword" - }, - - "telemetryD" : { - "type" : "keyword" - }, - "rawData" : { - "type" : "keyword" - }, - "telemetryF" : { - "type" : "keyword" - }, - "telemetryReportType" : { - "type" : "keyword" - }, - "telemetryI" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderPerhopMetadataLength" : { - "type" : "keyword" - }, - "telemetryQ" : { - "type" : "keyword" - }, - "telemetryDomainSpecificID" : { - "type" : "keyword" - }, - "udpLayersSize" : { - "type" : "keyword" - }, - "UDGDstPort" : { - "type" : "keyword" - }, - "IPvNextHopLimit" : { - "type" : "keyword" - }, - "telemetryDsMdBits" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderDomainSpecificID" : { - "type" : "keyword" - }, - "telemetryRepMdBits" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderRemainingHopCnt" : { - "type" : "keyword" - }, - "telemetrySequenceNumber" : { - "type" : "keyword" - }, - "telemetryMDLength" : { - "type" : "keyword" - }, - "IPvClass" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderD" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderE" : { - "type" : "keyword" - }, - "telemetryNodeId" : { - "type" : "keyword" - }, - "dataII" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderRes" : { - "type" : "keyword" - } - } - } - } - } -} diff --git a/playbooks/hcp/templates/tcp/tcp_data_parsing.json b/playbooks/hcp/templates/tcp/tcp_data_parsing.json deleted file mode 100644 index 4aad2c8f..00000000 --- a/playbooks/hcp/templates/tcp/tcp_data_parsing.json +++ /dev/null @@ -1,1332 +0,0 @@ -{ - "description" : "Transparent Security TCP v4 & v6 data parsing pipeline", - "processors" : [ - { - "set" : { - "field" : "ts.rawData", - "value" : "{{layers.data.data_data_data}}" - } - }, - { - "script" : { - "lang" : "painless", - "source" : "ctx.ts.telemetryReport =ctx.ts.rawData.substring(0, 48).replace(':','')", - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String hardwareIdStr = ctx.layers.data.data_data_data.replace(':','').substring(1, 3); - BigInteger hwdIdBigInt = new BigInteger(hardwareIdStr, 16); - BigInteger bitwiseAnd = new BigInteger('FC', 16); - BigInteger extractedHardwareId = hwdIdBigInt.and(bitwiseAnd); - String hwdIdStr = extractedHardwareId.toString(16); - ctx.ts.telemetryHwdId = hwdIdStr; - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String sequenceNumberPartStrI = tsRawData.substring(2, 3); - String sequenceNumberPartStrII = tsRawData.substring(3, 8); - String sequenceNumberPartStr = tsRawData.substring(2, 8); - BigInteger sequenceNumberBigInt = new BigInteger(sequenceNumberPartStr, 16); - BigInteger bitwiseAnd = new BigInteger('7FFFFFF', 16); - BigInteger updatedSequenceNumber = sequenceNumberBigInt.and(bitwiseAnd); - // String sequenceNumberHexStr = updatedSequenceNumber.toString(16); - ctx.ts.telemetrySequenceNumber = updatedSequenceNumber.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String nodeIdStr = tsRawData.substring(8, 16); - BigInteger nodeIdBigInt = new BigInteger(nodeIdStr, 16); - ctx.ts.telemetryNodeId =nodeIdBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String repTypeStr = tsRawData.substring(16, 17); - BigInteger repTypeBigInt = new BigInteger(repTypeStr, 16); - ctx.ts.telemetryReportType =repTypeBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String inTypeStr = tsRawData.substring(17, 18); - BigInteger inTypeBigInt = new BigInteger(inTypeStr, 16); - ctx.ts.telemetryInType = inTypeBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String reportLengthStr = tsRawData.substring(18, 20); - BigInteger reportLengthBigInt = new BigInteger(reportLengthStr, 16); - ctx.ts.telemetryReportLength = reportLengthBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String mdLengthStr = tsRawData.substring(20, 22); - BigInteger mdLengthBigInt = new BigInteger(mdLengthStr, 16); - ctx.ts.telemetryMDLength = mdLengthBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String dStr = tsRawData.substring(22, 23); - BigInteger dBigInt = new BigInteger(dStr, 16); - BigInteger bitwiseAnd = new BigInteger('8', 16); - BigInteger dRes = dBigInt.and(bitwiseAnd); - ctx.ts.telemetryD = dRes.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String qStr = tsRawData.substring(22, 23); - BigInteger qBigInt = new BigInteger(qStr, 16); - BigInteger bitwiseAnd = new BigInteger('4', 16); - BigInteger qRes = qBigInt.and(bitwiseAnd); - ctx.ts.telemetryQ = qRes.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String fStr = tsRawData.substring(22, 23); - BigInteger fBigInt = new BigInteger(fStr, 16); - BigInteger bitwiseAnd = new BigInteger('2', 16); - BigInteger fRes = fBigInt.and(bitwiseAnd); - ctx.ts.telemetryF = fRes.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String IStr = tsRawData.substring(22, 23); - BigInteger IBigInt = new BigInteger(IStr, 16); - BigInteger bitwiseAnd = new BigInteger('1', 16); - BigInteger IRes = IBigInt.and(bitwiseAnd); - ctx.ts.telemetryI = IRes.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String rsvdStr = tsRawData.substring(23, 24); - BigInteger rsvdBigInt = new BigInteger(rsvdStr, 16); - ctx.ts.telemetryRsvd = rsvdBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String repMdBitsStr = tsRawData.substring(24, 28); - BigInteger repMdBitsBigInt = new BigInteger(repMdBitsStr, 16); - ctx.ts.telemetryRepMdBits = repMdBitsBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String domainSpecificIDStr = tsRawData.substring(28, 32); - BigInteger domainSpecificIDBigInt = new BigInteger(domainSpecificIDStr, 16); - ctx.ts.telemetryDomainSpecificID = domainSpecificIDBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String dsMdBitsStr = tsRawData.substring(32, 36); - BigInteger dsMdBitsStrBigInt = new BigInteger(dsMdBitsStr, 16); - ctx.ts.telemetryDsMdBits = dsMdBitsStrBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String dsMdStatusStr = tsRawData.substring(36, 40); - BigInteger dsMdStatusBigInt = new BigInteger(dsMdStatusStr, 16); - ctx.ts.telemetryDsMdStatus = dsMdStatusBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String varOptStr = tsRawData.substring(40, 48); - BigInteger varOptBigInt = new BigInteger(varOptStr, 16); - ctx.ts.telemetryVarOpt = varOptBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.isIPv6 = true; - } else - { - ctx.ts.isIPv6 = false; - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.isIPv4 = true; - } else - { - ctx.ts.isIPv4 = false; - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String nextProtoStr = tsRawData.substring(138, 140); - BigInteger nextProtoBigInt = new BigInteger(nextProtoStr, 16); - ctx.ts.nextProto = nextProtoBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String nextProtoStr = tsRawData.substring(178, 180); - BigInteger nextProtoBigInt = new BigInteger(nextProtoStr, 16); - ctx.ts.nextProto = nextProtoBigInt.intValue(); - } - """, - "ignore_failure" : true - } - }, - - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - String tcpNextProtoStr = '6'; - int tcpNextProtoInt = Integer.parseInt(tcpNextProtoStr); - - if( etherType == '0800' && ipVersion == '4') - { - String nextProtoStr = tsRawData.substring(138, 140); - BigInteger nextProtoBigInt = new BigInteger(nextProtoStr, 16); - - if( ctx.ts.nextProto == tcpNextProtoInt ) - { - ctx.ts.isTCP = true; - }else { - ctx.ts.isTCP = false; - } - } - - if( etherType == '86dd' && ipVersion == '6') - { - String nextProtoStr = tsRawData.substring(178, 180); - BigInteger nextProtoBigInt = new BigInteger(nextProtoStr, 16); - String UDPNextProtoStr = ctx.ts.UDPIntShimHeaderNextProto.toString(); - if( ctx.ts.nextProto == tcpNextProtoInt ) - { - ctx.ts.isTCP = true; - }else { - ctx.ts.isTCP = false; - } - } - - """, - "ignore_failure" : true - } - }, - - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - String udpNextProtoStr = '17'; - int udpNextProtoInt = Integer.parseInt(udpNextProtoStr); - if( etherType == '0800' && ipVersion == '4') - { - String nextProtoStr = tsRawData.substring(138, 140); - BigInteger nextProtoBigInt = new BigInteger(nextProtoStr, 16); - String UDPNextProtoStr = ctx.ts.UDPIntShimHeaderNextProto.toString(); - if(ctx.ts.nextProto == udpNextProtoInt ) - { - ctx.ts.isUDP = true; - }else { - ctx.ts.isUDP = false; - } - } - - if( etherType == '86dd' && ipVersion == '6') - { - String nextProtoStr = tsRawData.substring(178, 180); - BigInteger nextProtoBigInt = new BigInteger(nextProtoStr, 16); - String UDPNextProtoStr = ctx.ts.UDPIntShimHeaderNextProto.toString(); - if(ctx.ts.nextProto == udpNextProtoInt ) - { - ctx.ts.isUDP = true; - }else { - ctx.ts.isUDP = false; - } - } - - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.data = tsRawData.substring(48, 252); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.data = tsRawData.substring(48, 292); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - ctx.ts.ethernetHeader = tsRawData.substring(48, 76); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - ctx.ts.ethernetHeaderDestMac = tsRawData.substring(48, 60); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - ctx.ts.ethernetHeaderSrcMac = tsRawData.substring(60, 72); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - ctx.ts.ethernetHeaderEtherType = tsRawData.substring(72, 76); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.IPvData = tsRawData.substring(76, 116); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.IPvData = tsRawData.substring(76, 156); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.IPVersion = tsRawData.substring(76, 77); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.IPVersion = tsRawData.substring(76, 77); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String IPv4SrcIPHexString = tsRawData.substring(100, 108); - String IPv4SrcIP = ""; - for(int i = 0; i < IPv4SrcIPHexString.length(); i = i + 2) { - IPv4SrcIP = IPv4SrcIP + Integer.valueOf(IPv4SrcIPHexString.substring(i, i+2), 16) + "."; - } - IPv4SrcIP = IPv4SrcIP.substring(0, IPv4SrcIP.length()-1); - ctx.ts.IPv4SrcIP = IPv4SrcIP; - } - if( etherType == '86dd' && ipVersion == '6') - { - String IPv6SrcIPHexString = tsRawData.substring(92,124); - String IPv6SrcIP = ""; - for(int i = 0; i < IPv6SrcIPHexString.length(); i = i + 2) { - IPv6SrcIP = IPv6SrcIP + Integer.valueOf(IPv6SrcIPHexString.substring(i, i+2), 16) + "."; - } - IPv6SrcIP = IPv6SrcIP.substring(0, IPv6SrcIP.length()-1); - ctx.ts.IPv6SrcIP = IPv6SrcIP; - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String IPv4DestIPHexString = tsRawData.substring(108, 116); - String IPv4DestIP = ""; - for(int i = 0; i < IPv4DestIPHexString.length(); i = i + 2) { - IPv4DestIP = IPv4DestIP + Integer.valueOf(IPv4DestIPHexString.substring(i, i+2), 16) + "."; - } - IPv4DestIP = IPv4DestIP.substring(0, IPv4DestIP.length()-1); - ctx.ts.IPv4DestIP = IPv4DestIP; - } - if( etherType == '86dd' && ipVersion == '6') - { - String IPv6DestIPHexString = tsRawData.substring(124,156); - String IPv6DestIP = ""; - for(int i = 0; i < IPv6DestIPHexString.length(); i = i + 2) { - IPv6DestIP = IPv6DestIP + Integer.valueOf(IPv6DestIPHexString.substring(i, i+2), 16) + "."; - } - IPv6DestIP = IPv6DestIP.substring(0, IPv6DestIP.length()-1); - ctx.ts.IPv6DestIP = IPv6DestIP; - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDGPData = tsRawData.substring(116, 132); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDGPData = tsRawData.substring(156, 172); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDGSrcPortStr = tsRawData.substring(116, 120); - BigInteger UDGSrcPortBigInt = new BigInteger(UDGSrcPortStr, 16); - ctx.ts.UDGSrcPort = UDGSrcPortBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDGSrcPortStr = tsRawData.substring(156, 160); - BigInteger UDGSrcPortBigInt = new BigInteger(UDGSrcPortStr, 16); - ctx.ts.UDGSrcPort = UDGSrcPortBigInt.intValue(); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDGDstPortStr = tsRawData.substring(120, 124); - BigInteger UDGDstPortBigInt = new BigInteger(UDGDstPortStr, 16); - ctx.ts.UDGDstPort = UDGDstPortBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDGDstPortStr = tsRawData.substring(160, 164); - BigInteger UDGDstPortBigInt = new BigInteger(UDGDstPortStr, 16); - ctx.ts.UDGDstPort = UDGDstPortBigInt.intValue(); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDGLengthStr = tsRawData.substring(124, 128); - BigInteger UDGLengthBigInt = new BigInteger(UDGLengthStr, 16); - ctx.ts.UDGLength = UDGLengthBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDGLengthStr = tsRawData.substring(164, 168); - BigInteger UDGLengthBigInt = new BigInteger(UDGLengthStr, 16); - ctx.ts.UDGLength = UDGLengthBigInt.intValue(); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDGChecksumStr = tsRawData.substring(128, 132); - BigInteger UDGChecksumBigInt = new BigInteger(UDGChecksumStr, 16); - ctx.ts.UDGChecksum = UDGChecksumBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDGChecksumStr = tsRawData.substring(168, 172); - BigInteger UDGChecksumBigInt = new BigInteger(UDGChecksumStr, 16); - ctx.ts.UDGChecksum = UDGChecksumBigInt.intValue(); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.InBandNetworkTelemetryData = tsRawData.substring(132, 196); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.InBandNetworkTelemetryData = tsRawData.substring(172, 236); - } - """, - "ignore_failure" : true - } - }, - - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDPIntShimHeaderData = tsRawData.substring(132, 140); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDPIntShimHeaderData = tsRawData.substring(172, 180); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDPIntShimHeaderType = tsRawData.substring(132, 133); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDPIntShimHeaderType = tsRawData.substring(172, 173); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String udpIntShimNPTStr = tsRawData.substring(133, 134); - BigInteger udpIntShimNPTBigInt = new BigInteger(udpIntShimNPTStr, 16); - BigInteger bitwiseAnd = new BigInteger('C', 16); - BigInteger result = udpIntShimNPTBigInt.and(bitwiseAnd); - ctx.ts.UDPIntShimHeaderNPT = result.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String udpIntShimNPTStr = tsRawData.substring(173, 174); - BigInteger udpIntShimNPTBigInt = new BigInteger(udpIntShimNPTStr, 16); - BigInteger bitwiseAnd = new BigInteger('C', 16); - BigInteger result = udpIntShimNPTBigInt.and(bitwiseAnd); - ctx.ts.UDPIntShimHeaderNPT = result.intValue(); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDPIntShimHeaderRes1 = tsRawData.substring(133, 134); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDPIntShimHeaderRes1 = tsRawData.substring(174, 175); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String lengthStr = tsRawData.substring(134, 136); - BigInteger lengthStrBigInt = new BigInteger(lengthStr, 16); - ctx.ts.UDPIntShimHeaderLength= lengthStrBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String lengthStr = tsRawData.substring(174, 176); - BigInteger lengthStrBigInt = new BigInteger(lengthStr, 16); - ctx.ts.UDPIntShimHeaderLength = lengthStrBigInt.intValue(); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDPIntShimHeaderRes2 = tsRawData.substring(136, 138); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDPIntShimHeaderRes2 = tsRawData.substring(176, 178); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String nextProtoStr = tsRawData.substring(138, 140); - BigInteger nextProtoBigInt = new BigInteger(nextProtoStr, 16); - ctx.ts.UDPIntShimHeaderNextProto = nextProtoBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String nextProtoStr = tsRawData.substring(178, 180); - BigInteger nextProtoBigInt = new BigInteger(nextProtoStr, 16); - ctx.ts.UDPIntShimHeaderNextProto = nextProtoBigInt.intValue(); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.INTMetadataHeaderData = tsRawData.substring(140, 164); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.INTMetadataHeaderData = tsRawData.substring(180, 204); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderVersionStr = tsRawData.substring(140, 141); - BigInteger UDPIntMetaDataHeaderVersionStrBigInt = new BigInteger(UDPIntMetaDataHeaderVersionStr, 16); - ctx.ts.UDPIntMetaDataHeaderVersion = UDPIntMetaDataHeaderVersionStrBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderVersionStr = tsRawData.substring(180, 181); - BigInteger UDPIntMetaDataHeaderVersionStrBigInt = new BigInteger(UDPIntMetaDataHeaderVersionStr, 16); - ctx.ts.UDPIntMetaDataHeaderVersion = UDPIntMetaDataHeaderVersionStrBigInt.intValue(); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderRes = tsRawData.substring(141, 142); - BigInteger UDPIntMetaDataHeaderResBigInt = new BigInteger(UDPIntMetaDataHeaderRes, 16); - BigInteger bitwiseAnd = new BigInteger('C', 16); - BigInteger result = UDPIntMetaDataHeaderResBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderRes = result.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderRes = tsRawData.substring(181, 182); - BigInteger UDPIntMetaDataHeaderResBigInt = new BigInteger(UDPIntMetaDataHeaderRes, 16); - BigInteger bitwiseAnd = new BigInteger('C', 16); - BigInteger result = UDPIntMetaDataHeaderResBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderRes = result.intValue(); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderD = tsRawData.substring(141, 142); - BigInteger UDPIntMetaDataHeaderDBigInt = new BigInteger(UDPIntMetaDataHeaderD, 16); - BigInteger bitwiseAnd = new BigInteger('2', 16); - BigInteger result = UDPIntMetaDataHeaderDBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderD = result.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderD = tsRawData.substring(181, 182); - BigInteger UDPIntMetaDataHeaderDBigInt = new BigInteger(UDPIntMetaDataHeaderD, 16); - BigInteger bitwiseAnd = new BigInteger('2', 16); - BigInteger result = UDPIntMetaDataHeaderDBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderD = result.intValue(); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderE = tsRawData.substring(141, 142); - BigInteger UDPIntMetaDataHeaderEBigInt = new BigInteger(UDPIntMetaDataHeaderE, 16); - BigInteger bitwiseAnd = new BigInteger('1', 16); - BigInteger result = UDPIntMetaDataHeaderEBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderE = result.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderE = tsRawData.substring(181, 182); - BigInteger UDPIntMetaDataHeaderEBigInt = new BigInteger(UDPIntMetaDataHeaderE, 16); - BigInteger bitwiseAnd = new BigInteger('1', 16); - BigInteger result = UDPIntMetaDataHeaderEBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderE = result.intValue(); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderM = tsRawData.substring(142, 143); - BigInteger UDPIntMetaDataHeaderMBigInt = new BigInteger(UDPIntMetaDataHeaderM, 16); - BigInteger bitwiseAnd = new BigInteger('8', 16); - BigInteger result = UDPIntMetaDataHeaderMBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderM = result.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderM = tsRawData.substring(182, 183); - BigInteger UDPIntMetaDataHeaderMBigInt = new BigInteger(UDPIntMetaDataHeaderM, 16); - BigInteger bitwiseAnd = new BigInteger('8', 16); - BigInteger result = UDPIntMetaDataHeaderMBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderM = result.intValue(); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderPerhopMetadataLengthStr = tsRawData.substring(144, 146); - BigInteger UDPIntMetaDataHeaderPerhopMetadataLengthBigInt = new BigInteger(UDPIntMetaDataHeaderPerhopMetadataLengthStr, 16); - BigInteger bitwiseAnd = new BigInteger('1F', 16); - BigInteger result = UDPIntMetaDataHeaderPerhopMetadataLengthBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderPerhopMetadataLength = result.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderPerhopMetadataLengthStr = tsRawData.substring(184, 186); - BigInteger UDPIntMetaDataHeaderPerhopMetadataLengthBigInt = new BigInteger(UDPIntMetaDataHeaderPerhopMetadataLengthStr, 16); - BigInteger bitwiseAnd = new BigInteger('1F', 16); - BigInteger result = UDPIntMetaDataHeaderPerhopMetadataLengthBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderPerhopMetadataLength = result.intValue(); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderRemainingHopCntStr = tsRawData.substring(146, 148); - BigInteger UDPIntMetaDataHeaderRemainingHopCntBigInt = new BigInteger(UDPIntMetaDataHeaderRemainingHopCntStr, 16); - ctx.ts.UDPIntMetaDataHeaderRemainingHopCnt = UDPIntMetaDataHeaderRemainingHopCntBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderRemainingHopCntStr = tsRawData.substring(186, 188); - BigInteger UDPIntMetaDataHeaderRemainingHopCntBigInt = new BigInteger(UDPIntMetaDataHeaderRemainingHopCntStr, 16); - ctx.ts.UDPIntMetaDataHeaderRemainingHopCnt = UDPIntMetaDataHeaderRemainingHopCntBigInt.intValue(); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderInstructionBitmapStr = tsRawData.substring(148, 152); - BigInteger UDPIntMetaDataHeaderInstructionBitmapBigInt = new BigInteger(UDPIntMetaDataHeaderInstructionBitmapStr, 16); - ctx.ts.UDPIntMetaDataHeaderInstructionBitmap = UDPIntMetaDataHeaderInstructionBitmapBigInt.toString(16); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderInstructionBitmapStr = tsRawData.substring(188, 192); - BigInteger UDPIntMetaDataHeaderInstructionBitmapBigInt = new BigInteger(UDPIntMetaDataHeaderInstructionBitmapStr, 16); - ctx.ts.UDPIntMetaDataHeaderInstructionBitmap = UDPIntMetaDataHeaderInstructionBitmapBigInt.toString(16); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderDomainSpecificIDStr = tsRawData.substring(152, 156); - BigInteger UDPIntMetaDataHeaderDomainSpecificIDBigInt = new BigInteger(UDPIntMetaDataHeaderDomainSpecificIDStr, 16); - ctx.ts.UDPIntMetaDataHeaderDomainSpecificID = UDPIntMetaDataHeaderDomainSpecificIDBigInt.toString(16); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderDomainSpecificIDStr = tsRawData.substring(192, 196); - BigInteger UDPIntMetaDataHeaderDomainSpecificIDBigInt = new BigInteger(UDPIntMetaDataHeaderDomainSpecificIDStr, 16); - ctx.ts.UDPIntMetaDataHeaderDomainSpecificID = UDPIntMetaDataHeaderDomainSpecificIDBigInt.toString(16); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderDSInstructionStr = tsRawData.substring(156, 160); - BigInteger UDPIntMetaDataHeaderDSInstructionBigInt = new BigInteger(UDPIntMetaDataHeaderDSInstructionStr, 16); - ctx.ts.UDPIntMetaDataHeaderDSInstruction = UDPIntMetaDataHeaderDSInstructionBigInt.toString(16); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderDSInstructionStr = tsRawData.substring(196, 200); - BigInteger UDPIntMetaDataHeaderDSInstructionBigInt = new BigInteger(UDPIntMetaDataHeaderDSInstructionStr, 16); - ctx.ts.UDPIntMetaDataHeaderDSInstruction = UDPIntMetaDataHeaderDSInstructionBigInt.toString(16); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderDSFlagsStr = tsRawData.substring(160, 164); - BigInteger UDPIntMetaDataHeaderDSFlagsBigInt = new BigInteger(UDPIntMetaDataHeaderDSFlagsStr, 16); - ctx.ts.UDPIntMetaDataHeaderDSFlags = UDPIntMetaDataHeaderDSFlagsBigInt.toString(16); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderDSFlagsStr = tsRawData.substring(200, 204); - BigInteger UDPIntMetaDataHeaderDSFlagsBigInt = new BigInteger(UDPIntMetaDataHeaderDSFlagsStr, 16); - ctx.ts.UDPIntMetaDataHeaderDSFlags = UDPIntMetaDataHeaderDSFlagsBigInt.toString(16); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.INTMetadataStackData = tsRawData.substring(164, 196); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.INTMetadataStackData = tsRawData.substring(204, 236); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.INTMetadataStackSwitchID = tsRawData.substring(164, 172); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.INTMetadataStackSwitchID = tsRawData.substring(204, 212); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.INTMetadataSourceMetadata = tsRawData.substring(172, 196); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.INTMetadataSourceMetadata = tsRawData.substring(212, 236); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.INTMetadataSourceMetadataSwitchId = tsRawData.substring(172, 180); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.INTMetadataSourceMetadataSwitchId = tsRawData.substring(212, 220); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String rawOriginatingMac = tsRawData.substring(180, 192); - StringBuilder formattedOriginatingMac = new StringBuilder(); - for(int i = 0; i < rawOriginatingMac.length();) - { - formattedOriginatingMac.append(rawOriginatingMac.substring(i++, ++i)); - if(i < rawOriginatingMac.length()) formattedOriginatingMac.append(":"); - } - ctx.ts.INTMetadataSourceMetadataOriginatingMac = formattedOriginatingMac.toString(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String rawOriginatingMac = tsRawData.substring(220, 232); - StringBuilder formattedOriginatingMac = new StringBuilder(); - for(int i = 0; i < rawOriginatingMac.length();) - { - formattedOriginatingMac.append(rawOriginatingMac.substring(i++, ++i)); - if(i < rawOriginatingMac.length()) formattedOriginatingMac.append(":"); - } - ctx.ts.INTMetadataSourceMetadataOriginatingMac = formattedOriginatingMac.toString(); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.INTMetadataSourceMetadataReserved = tsRawData.substring(192, 196); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.INTMetadataSourceMetadataReserved = tsRawData.substring(232, 236); - } - """, - "ignore_failure" : true - } - }, - - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - String tcpNextProtoStr = '6'; - int tcpNextProtoInt = Integer.parseInt(tcpNextProtoStr); - if( etherType == '0800' && ipVersion == '4' && ctx.ts.nextProto == tcpNextProtoInt ) - { - ctx.ts.TCPData = tsRawData.substring(196, 236); - } - if(etherType == '86dd' && ipVersion == '6' && ctx.ts.nextProto == tcpNextProtoInt ) - { - ctx.ts.TCPData = tsRawData.substring(236, 276); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - String tcpNextProtoStr = '6'; - int tcpNextProtoInt = Integer.parseInt(tcpNextProtoStr); - if( etherType == '0800' && ipVersion == '4' && ctx.ts.nextProto == tcpNextProtoInt ) - { - String TCPSrcPortStr = tsRawData.substring(196, 200); - BigInteger TCPSrcPortBigInt = new BigInteger(TCPSrcPortStr,16); - ctx.ts.TCPSrcPort = TCPSrcPortBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6' && ctx.ts.nextProto == tcpNextProtoInt ) - { - String TCPSrcPortStr = tsRawData.substring(236, 240); - BigInteger TCPSrcPortBigInt = new BigInteger(TCPSrcPortStr,16); - ctx.ts.TCPSrcPort = TCPSrcPortBigInt.intValue(); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - String tcpNextProtoStr = '6'; - int tcpNextProtoInt = Integer.parseInt(tcpNextProtoStr); - if( etherType == '0800' && ipVersion == '4' && ctx.ts.nextProto == tcpNextProtoInt ) - { - String TCPDstPortStr = tsRawData.substring(200, 204); - BigInteger TCPDstPortBigInt = new BigInteger(TCPDstPortStr,16); - ctx.ts.TCPDestPort = TCPDstPortBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6' && ctx.ts.nextProto == tcpNextProtoInt ) - { - String TCPDstPortStr = tsRawData.substring(240, 244); - BigInteger TCPDstPortBigInt = new BigInteger(TCPDstPortStr,16); - ctx.ts.TCPDestPort = TCPDstPortBigInt.intValue(); - } - """, - "ignore_failure" : true - } - }, - - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - String tcpNextProtoStr = '6'; - int tcpNextProtoInt = Integer.parseInt(tcpNextProtoStr); - if( etherType == '0800' && ipVersion == '4' && ctx.ts.nextProto == tcpNextProtoInt) - { - ctx.ts.dataII = tsRawData.substring(236, 288); - } - if(etherType == '86dd' && ipVersion == '6' && ctx.ts.nextProto == tcpNextProtoInt) - { - ctx.ts.dataII = tsRawData.substring(252, 304); - } - """, - "ignore_failure" : true - } - } - ] -} diff --git a/playbooks/hcp/templates/tcp/tcp_monitor_def.json b/playbooks/hcp/templates/tcp/tcp_monitor_def.json deleted file mode 100644 index 71061d74..00000000 --- a/playbooks/hcp/templates/tcp/tcp_monitor_def.json +++ /dev/null @@ -1,156 +0,0 @@ -{ - "type": "monitor", - "name": "DDOS-TCP-Monitor", - "enabled": true, - "schedule": { - "period": { - "interval": 1, - "unit": "MINUTES" - } - }, - "inputs": [{ - "search": { - "indices": ["packets-*"], - "query": { - "size": 0, - "query": { - "bool": { - "filter": [ - { - "range": { - "timestamp": { - "from": "{{period_end}}||-1m", - "to": "{{period_end}}", - "include_lower": true, - "include_upper": true, - "format": "epoch_millis", - "boost": 1 - } - } - }, - { - "wildcard": { - "ts.UDP2DstPort": { - "wildcard": "*5792*", - "boost": 1 - } - } - } - ], - "adjust_pure_negative": true, - "boost": 1 - } - }, - "_source": { - "includes": [ - "ts.TCPDestPort", - "ts.IPvDestAddr", - "ts.INTMetadataSourceMetadataOriginatingMac", - "ts.IPv4SrcIP", - "ts.IPv4DestIP", - "ts.IPv6SrcIP", - "ts.IPv6DestIP" - ], - "excludes": [] - }, - "aggregations": { - "TCPDestPortValueCount": { - "value_count": { - "field": "ts.TCPDestPort" - } - }, - "IPv4SrcIP": { - "terms": { - "field": "ts.IPv4SrcIP", - "size": 10, - "min_doc_count": 1, - "shard_min_doc_count": 0, - "show_term_doc_count_error": false, - "order": [ - { - "_count": "desc" - }, - { - "_key": "asc" - } - ] - } - }, - "IPv4DestIP": { - "terms": { - "field": "ts.IPv4DestIP", - "size": 10, - "min_doc_count": 1, - "shard_min_doc_count": 0, - "show_term_doc_count_error": false, - "order": [ - { - "_count": "desc" - }, - { - "_key": "asc" - } - ] - } - }, - "TCPDestPortValue": { - "terms": { - "field": "ts.TCPDestPort", - "size": 10, - "min_doc_count": 1, - "shard_min_doc_count": 0, - "show_term_doc_count_error": false, - "order": [ - { - "_count": "desc" - }, - { - "_key": "asc" - } - ] - } - }, - "OriginatingMacAddrTCPv4": { - "terms": { - "field": "ts.INTMetadataSourceMetadataOriginatingMac", - "size": 10, - "min_doc_count": 1, - "shard_min_doc_count": 0, - "show_term_doc_count_error": false, - "order": [ - { - "_count": "desc" - }, - { - "_key": "asc" - } - ] - } - } - } - } - } - }], - "triggers": [{ - "name": "DDOS-Trigger", - "severity": "1", - "condition": { - "script": { - "source": "ctx.results[0].aggregations.TCPDestPortValueCount.value >= 10", - "lang": "painless" - } - }, - "actions": [{ - "name": "Sdn-WebHook-Action", - "destination_id": "sdnWebhookResponseId", - "message_template": { - "source":"{\"event_action\":\"trigger\", \"dst_ip\":\"{{ctx.results.0.aggregations.IPv4DestIP.buckets.0.key}}\",\"dst_port\":\"{{ctx.results.0.aggregations.TCPDestPortValue.buckets.0.key}}\",\"src_ip\":\"{{ctx.results.0.aggregations.IPv4SrcIP.buckets.0.key}}\",\"src_mac\":\"{{ctx.results.0.aggregations.OriginatingMacAddrTCPv4.buckets.0.key}}\"}", - "lang": "mustache", - "options": { - "content_type": "application/json" - } - }, - "throttle_enabled": false - }] - }] -} diff --git a/playbooks/hcp/templates/udp/udp_data_mapping.json b/playbooks/hcp/templates/udp/udp_data_mapping.json deleted file mode 100644 index 24e756d9..00000000 --- a/playbooks/hcp/templates/udp/udp_data_mapping.json +++ /dev/null @@ -1,315 +0,0 @@ -{ - "order" : 0, - "index_patterns" : [ - "packets-*" - ], - "settings" : { - "index" : { - "number_of_replicas" : "0", - "default_pipeline" : "ts_parsing", - "max_script_fields" : "200" - } - }, - "mappings" : { - "dynamic" : "false", - "properties" : { - "layers" : { - "properties" : { - "data_raw" : { - "type" : "keyword" - }, - "data" : { - "properties" : { - "data_data_data_raw" : { - "type" : "keyword" - }, - "data_data_len" : { - "type" : "keyword" - }, - "data_data_data" : { - "type" : "keyword" - } - } - } - } - }, - "timestamp" : { - "type" : "date" - }, - "ts" : { - "properties" : { - "INTMetadataSourceMetadataReserved" : { - "type" : "keyword" - }, - "UDP2SrcPort" : { - "type" : "keyword" - }, - "UDPIntShimHeaderType" : { - "type" : "keyword" - }, - "telemetryReportLength" : { - "type" : "keyword" - }, - "data" : { - "type" : "keyword" - }, - "ethernetHeaderDestMac" : { - "type" : "keyword" - }, - "UDGSrcPort" : { - "type" : "keyword" - }, - "telemetryRsvd" : { - "type" : "keyword" - }, - "INTMetadataSourceMetadataSwitchId" : { - "type" : "keyword" - }, - "UDPIntShimHeaderNextProto" : { - "type" : "keyword" - }, - "IPv6SrcIP" : { - "type" : "keyword" - }, - "ethernetHeaderSrcMac" : { - "type" : "keyword" - }, - "isIPv4" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderVersion" : { - "type" : "keyword" - }, - "IPvNextHdrProto" : { - "type" : "keyword" - }, - "telemetryReportVersion" : { - "type" : "keyword" - }, - "isIPv6" : { - "type" : "keyword" - }, - "INTMetadataSourceMetadataOriginatingMac" : { - "type" : "keyword" - }, - "UDP2Checksum" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderInstructionBitmap" : { - "type" : "keyword" - }, - "INTMetadataSourceMetadata" : { - "type" : "keyword" - }, - "IPvSrcAddr" : { - "type" : "keyword" - }, - "UDGPData" : { - "type" : "keyword" - }, - "tsRawDataSize" : { - "type" : "keyword" - }, - "UDGChecksum" : { - "type" : "keyword" - }, - "IPvDestAddr" : { - "type" : "keyword" - }, - "INTMetadataHeaderData" : { - "type" : "keyword" - }, - "telemetryInType" : { - "type" : "keyword" - }, - "telemetryReportVarOptTimeStamp" : { - "type" : "keyword" - }, - "dropTelemetryReport" : { - "type" : "keyword" - }, - "telemetryReport" : { - "type" : "keyword" - }, - "telemetryDsMdStatus" : { - "type" : "keyword" - }, - "ts.isIPv4":{ - "type" : "keyword" - }, - "ts.isIPv6":{ - "type" : "keyword" - }, - "UDPIntMetaDataHeaderDSFlags" : { - "type" : "keyword" - }, - "InBandNetworkTelemetryData" : { - "type" : "keyword" - }, - "IPv4SrcIP" : { - "type" : "keyword" - }, - "UDPIntShimHeaderRes2" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderDSInstruction" : { - "type" : "keyword" - }, - "UDP2Data" : { - "type" : "keyword" - }, - "UDPIntShimHeaderRes1" : { - "type" : "keyword" - }, - "telemetryHwdId" : { - "type" : "keyword" - }, - "UDP2Length" : { - "type" : "keyword" - }, - "telemetryVarOpt" : { - "type" : "keyword" - }, - "ethernetHeaderEtherType" : { - "type" : "keyword" - }, - "IPvData" : { - "type" : "keyword" - }, - "INTMetadataStackData" : { - "type" : "keyword" - }, - "IPvFlow" : { - "type" : "keyword" - }, - "UDPIntShimHeaderData" : { - "type" : "keyword" - }, - "INTMetadataStackSwitchID" : { - "type" : "keyword" - }, - "IPVersion" : { - "type" : "keyword" - }, - - "IPvPayloadLength" : { - "type" : "keyword" - }, - "IPv4DestIP" : { - "type" : "keyword" - }, - "UDPIntShimHeaderLength" : { - "type" : "keyword" - }, - "UDPIntShimHeaderNPT" : { - "type" : "keyword" - }, - "UDGLength" : { - "type" : "keyword" - }, - "telemetryReportVarOptDropCount" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderM" : { - "type" : "keyword" - }, - "UDP2DstPort" : { - "type" : "keyword" - }, - "telemetryD" : { - "type" : "keyword" - }, - "dropReportHash" : { - "type" : "keyword" - }, - - "rawData" : { - "type" : "keyword" - }, - "telemetryF" : { - "type" : "keyword" - }, - "telemetryReportType" : { - "type" : "keyword" - }, - "telemetryI" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderPerhopMetadataLength" : { - "type" : "keyword" - }, - "telemetryQ" : { - "type" : "keyword" - }, - "telemetryDomainSpecificID" : { - "type" : "keyword" - }, - "udpLayersSize" : { - "type" : "keyword" - }, - "UDGDstPort" : { - "type" : "keyword" - }, - "IPvNextHopLimit" : { - "type" : "keyword" - }, - "dropHashInput" : { - "type" : "keyword" - }, - "telemetryDsMdBits" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderDomainSpecificID" : { - "type" : "keyword" - }, - "telemetryReportVarOpBsmd" : { - "type" : "keyword" - }, - "telemetryRepMdBits" : { - "type" : "keyword" - }, - "telemetryReportVarOptHash" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderRemainingHopCnt" : { - "type" : "keyword" - }, - "IPv6DestIP" : { - "type" : "keyword" - }, - "isDropReport" : { - "type" : "keyword" - }, - "IPvClass" : { - "type" : "keyword" - }, - "telemetryMDLength" : { - "type" : "keyword" - }, - "telemetrySequenceNumber" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderD" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderE" : { - "type" : "keyword" - }, - "telemetryNodeId" : { - "type" : "keyword" - }, - "sha256Hash" : { - "type" : "keyword" - }, - "dataII" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderRes" : { - "type" : "keyword" - } - } - } - } - }, - "aliases" : { } - } diff --git a/playbooks/hcp/templates/udp/udp_data_parsing.json b/playbooks/hcp/templates/udp/udp_data_parsing.json deleted file mode 100644 index a04d5f31..00000000 --- a/playbooks/hcp/templates/udp/udp_data_parsing.json +++ /dev/null @@ -1,1536 +0,0 @@ -{ - "description" : "Transparent Security UDP v4 & v6 data parsing pipeline", - "processors" : [ - { - "set" : { - "field" : "ts.rawData", - "value" : "{{layers.data.data_data_data}}" - } - }, - - { - "script" : { - "lang" : "painless", - "source" : "ctx.ts.telemetryReport = ctx.ts.rawData.substring(0, 48).replace(':','')", - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : "ctx.ts.dropTelemetryReport = ctx.ts.rawData.substring(0, 200).replace(':','')", - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String hardwareIdStr = ctx.layers.data.data_data_data.replace(':','').substring(1, 3); - BigInteger hwdIdBigInt = new BigInteger(hardwareIdStr, 16); - BigInteger bitwiseAnd = new BigInteger('FC', 16); - BigInteger extractedHardwareId = hwdIdBigInt.and(bitwiseAnd); - String hwdIdStr = extractedHardwareId.toString(16); - ctx.ts.telemetryHwdId = hwdIdStr; - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String sequenceNumberPartStrI = tsRawData.substring(2, 3); - String sequenceNumberPartStrII = tsRawData.substring(3, 8); - String sequenceNumberPartStr = tsRawData.substring(2, 8); - BigInteger sequenceNumberBigInt = new BigInteger(sequenceNumberPartStr, 16); - BigInteger bitwiseAnd = new BigInteger('7FFFFFF', 16); - BigInteger updatedSequenceNumber = sequenceNumberBigInt.and(bitwiseAnd); - // String sequenceNumberHexStr = updatedSequenceNumber.toString(16); - ctx.ts.telemetrySequenceNumber = updatedSequenceNumber.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String nodeIdStr = tsRawData.substring(8, 16); - BigInteger nodeIdBigInt = new BigInteger(nodeIdStr, 16); - ctx.ts.telemetryNodeId =nodeIdBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String repTypeStr = tsRawData.substring(16, 17); - BigInteger repTypeBigInt = new BigInteger(repTypeStr, 16); - ctx.ts.telemetryReportType =repTypeBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String inTypeStr = tsRawData.substring(17, 18); - BigInteger inTypeBigInt = new BigInteger(inTypeStr, 16); - ctx.ts.telemetryInType = inTypeBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String reportLengthStr = tsRawData.substring(18, 20); - BigInteger reportLengthBigInt = new BigInteger(reportLengthStr, 16); - ctx.ts.telemetryReportLength = reportLengthBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String mdLengthStr = tsRawData.substring(20, 22); - BigInteger mdLengthBigInt = new BigInteger(mdLengthStr, 16); - ctx.ts.telemetryMDLength = mdLengthBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String dStr = tsRawData.substring(22, 23); - BigInteger dBigInt = new BigInteger(dStr, 16); - BigInteger bitwiseAnd = new BigInteger('8', 16); - BigInteger dRes = dBigInt.and(bitwiseAnd); - ctx.ts.telemetryD = dRes.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String qStr = tsRawData.substring(22, 23); - BigInteger qBigInt = new BigInteger(qStr, 16); - BigInteger bitwiseAnd = new BigInteger('4', 16); - BigInteger qRes = qBigInt.and(bitwiseAnd); - ctx.ts.telemetryQ = qRes.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String fStr = tsRawData.substring(22, 23); - BigInteger fBigInt = new BigInteger(fStr, 16); - BigInteger bitwiseAnd = new BigInteger('2', 16); - BigInteger fRes = fBigInt.and(bitwiseAnd); - ctx.ts.telemetryF = fRes.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String IStr = tsRawData.substring(22, 23); - BigInteger IBigInt = new BigInteger(IStr, 16); - BigInteger bitwiseAnd = new BigInteger('1', 16); - BigInteger IRes = IBigInt.and(bitwiseAnd); - ctx.ts.telemetryI = IRes.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String rsvdStr = tsRawData.substring(23, 24); - BigInteger rsvdBigInt = new BigInteger(rsvdStr, 16); - ctx.ts.telemetryRsvd = rsvdBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String repMdBitsStr = tsRawData.substring(24, 28); - BigInteger repMdBitsBigInt = new BigInteger(repMdBitsStr, 16); - ctx.ts.telemetryRepMdBits = repMdBitsBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String domainSpecificIDStr = tsRawData.substring(28, 32); - BigInteger domainSpecificIDBigInt = new BigInteger(domainSpecificIDStr, 16); - ctx.ts.telemetryDomainSpecificID = domainSpecificIDBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String dsMdBitsStr = tsRawData.substring(32, 36); - BigInteger dsMdBitsStrBigInt = new BigInteger(dsMdBitsStr, 16); - ctx.ts.telemetryDsMdBits = dsMdBitsStrBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String dsMdStatusStr = tsRawData.substring(36, 40); - BigInteger dsMdStatusBigInt = new BigInteger(dsMdStatusStr, 16); - ctx.ts.telemetryDsMdStatus = dsMdStatusBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String varOptStr = tsRawData.substring(40, 48); - BigInteger varOptBigInt = new BigInteger(varOptStr, 16); - ctx.ts.telemetryVarOpt = varOptBigInt.intValue(); - } - - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.isIPv6 = true; - } else - { - ctx.ts.isIPv6 = false; - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - String dropReportStrValue = '2'; - String telemetryIPv4StrValue ='4'; - String telemetryIPv6StrValue ='5'; - String telemetryInTypeStrValue = ctx.ts.telemetryInType.toString(); - if(dropReportStrValue == telemetryInTypeStrValue) - { - ctx.ts.isDropReport = true; - } - if( (telemetryInTypeStrValue == telemetryIPv4StrValue) || (telemetryInTypeStrValue == telemetryIPv6StrValue) ) - { - ctx.ts.isDropReport = false; - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.isIPv4 = true; - } else - { - ctx.ts.isIPv4 = false; - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.data = tsRawData.substring(48, 252); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.data = tsRawData.substring(48, 292); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - ctx.ts.ethernetHeader = tsRawData.substring(48, 76); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - ctx.ts.ethernetHeaderDestMac = tsRawData.substring(48, 60); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - ctx.ts.ethernetHeaderSrcMac = tsRawData.substring(60, 72); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - ctx.ts.ethernetHeaderEtherType = tsRawData.substring(72, 76); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.IPvData = tsRawData.substring(76, 116); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.IPvData = tsRawData.substring(76, 156); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.IPVersion = tsRawData.substring(76, 77); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.IPVersion = tsRawData.substring(76, 77); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String IPv4SrcIPHexString = tsRawData.substring(100, 108); - String IPv4SrcIP = ""; - for(int i = 0; i < IPv4SrcIPHexString.length(); i = i + 2) { - IPv4SrcIP = IPv4SrcIP + Integer.valueOf(IPv4SrcIPHexString.substring(i, i+2), 16) + "."; - } - IPv4SrcIP = IPv4SrcIP.substring(0, IPv4SrcIP.length()-1); - ctx.ts.IPv4SrcIP = IPv4SrcIP; - } - if( etherType == '86dd' && ipVersion == '6') - { - String IPv6SrcIPHexString = tsRawData.substring(92,124); - String IPv6SrcIP = ""; - for(int i = 0; i < IPv6SrcIPHexString.length(); i = i + 4) { - IPv6SrcIP = IPv6SrcIP + IPv6SrcIPHexString.substring(i, i+4) + ":"; - } - IPv6SrcIP = IPv6SrcIP.substring(0, IPv6SrcIP.length()-1); - ctx.ts.IPv6SrcIP = IPv6SrcIP; - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String IPv4DestIPHexString = tsRawData.substring(108, 116); - String IPv4DestIP = ""; - for(int i = 0; i < IPv4DestIPHexString.length(); i = i + 2) { - IPv4DestIP = IPv4DestIP + Integer.valueOf(IPv4DestIPHexString.substring(i, i+2), 16) + "."; - } - IPv4DestIP = IPv4DestIP.substring(0, IPv4DestIP.length()-1); - ctx.ts.IPv4DestIP = IPv4DestIP; - } - if( etherType == '86dd' && ipVersion == '6') - { - String IPv6DestIPHexString = tsRawData.substring(124,156); - String IPv6DestIP = ""; - for(int i = 0; i < IPv6DestIPHexString.length(); i = i + 4) { - IPv6DestIP = IPv6DestIP + IPv6DestIPHexString.substring(i, i+4) + ":"; - } - IPv6DestIP = IPv6DestIP.substring(0, IPv6DestIP.length()-1); - ctx.ts.IPv6DestIP = IPv6DestIP; - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDGPData = tsRawData.substring(116, 132); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDGPData = tsRawData.substring(156, 172); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDGSrcPortStr = tsRawData.substring(116, 120); - BigInteger UDGSrcPortBigInt = new BigInteger(UDGSrcPortStr, 16); - ctx.ts.UDGSrcPort = UDGSrcPortBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDGSrcPortStr = tsRawData.substring(156, 160); - BigInteger UDGSrcPortBigInt = new BigInteger(UDGSrcPortStr, 16); - ctx.ts.UDGSrcPort = UDGSrcPortBigInt.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDGDstPortStr = tsRawData.substring(120, 124); - BigInteger UDGDstPortBigInt = new BigInteger(UDGDstPortStr, 16); - ctx.ts.UDGDstPort = UDGDstPortBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDGDstPortStr = tsRawData.substring(160, 164); - BigInteger UDGDstPortBigInt = new BigInteger(UDGDstPortStr, 16); - ctx.ts.UDGDstPort = UDGDstPortBigInt.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDGLengthStr = tsRawData.substring(124, 128); - BigInteger UDGLengthBigInt = new BigInteger(UDGLengthStr, 16); - ctx.ts.UDGLength = UDGLengthBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDGLengthStr = tsRawData.substring(164, 168); - BigInteger UDGLengthBigInt = new BigInteger(UDGLengthStr, 16); - ctx.ts.UDGLength = UDGLengthBigInt.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDGChecksumStr = tsRawData.substring(128, 132); - BigInteger UDGChecksumBigInt = new BigInteger(UDGChecksumStr, 16); - ctx.ts.UDGChecksum = UDGChecksumBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDGChecksumStr = tsRawData.substring(168, 172); - BigInteger UDGChecksumBigInt = new BigInteger(UDGChecksumStr, 16); - ctx.ts.UDGChecksum = UDGChecksumBigInt.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.InBandNetworkTelemetryData = tsRawData.substring(132, 196); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.InBandNetworkTelemetryData = tsRawData.substring(172, 236); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDPIntShimHeaderData = tsRawData.substring(132, 140); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDPIntShimHeaderData = tsRawData.substring(172, 180); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDPIntShimHeaderType = tsRawData.substring(132, 133); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDPIntShimHeaderType = tsRawData.substring(172, 173); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String udpIntShimNPTStr = tsRawData.substring(133, 134); - BigInteger udpIntShimNPTBigInt = new BigInteger(udpIntShimNPTStr, 16); - BigInteger bitwiseAnd = new BigInteger('C', 16); - BigInteger result = udpIntShimNPTBigInt.and(bitwiseAnd); - ctx.ts.UDPIntShimHeaderNPT = result.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String udpIntShimNPTStr = tsRawData.substring(173, 174); - BigInteger udpIntShimNPTBigInt = new BigInteger(udpIntShimNPTStr, 16); - BigInteger bitwiseAnd = new BigInteger('C', 16); - BigInteger result = udpIntShimNPTBigInt.and(bitwiseAnd); - ctx.ts.UDPIntShimHeaderNPT = result.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDPIntShimHeaderRes1 = tsRawData.substring(133, 134); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDPIntShimHeaderRes1 = tsRawData.substring(174, 175); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String lengthStr = tsRawData.substring(134, 136); - BigInteger lengthStrBigInt = new BigInteger(lengthStr, 16); - ctx.ts.UDPIntShimHeaderLength= lengthStrBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String lengthStr = tsRawData.substring(174, 176); - BigInteger lengthStrBigInt = new BigInteger(lengthStr, 16); - ctx.ts.UDPIntShimHeaderLength = lengthStrBigInt.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDPIntShimHeaderRes2 = tsRawData.substring(136, 138); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDPIntShimHeaderRes2 = tsRawData.substring(176, 178); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String nextProtoStr = tsRawData.substring(138, 140); - BigInteger nextProtoBigInt = new BigInteger(nextProtoStr, 16); - ctx.ts.UDPIntShimHeaderNextProto = nextProtoBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String nextProtoStr = tsRawData.substring(178, 180); - BigInteger nextProtoBigInt = new BigInteger(nextProtoStr, 16); - ctx.ts.UDPIntShimHeaderNextProto = nextProtoBigInt.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.INTMetadataHeaderData = tsRawData.substring(140, 164); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.INTMetadataHeaderData = tsRawData.substring(180, 204); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderVersionStr = tsRawData.substring(140, 141); - BigInteger UDPIntMetaDataHeaderVersionStrBigInt = new BigInteger(UDPIntMetaDataHeaderVersionStr, 16); - ctx.ts.UDPIntMetaDataHeaderVersion = UDPIntMetaDataHeaderVersionStrBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderVersionStr = tsRawData.substring(180, 181); - BigInteger UDPIntMetaDataHeaderVersionStrBigInt = new BigInteger(UDPIntMetaDataHeaderVersionStr, 16); - ctx.ts.UDPIntMetaDataHeaderVersion = UDPIntMetaDataHeaderVersionStrBigInt.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderRes = tsRawData.substring(141, 142); - BigInteger UDPIntMetaDataHeaderResBigInt = new BigInteger(UDPIntMetaDataHeaderRes, 16); - BigInteger bitwiseAnd = new BigInteger('C', 16); - BigInteger result = UDPIntMetaDataHeaderResBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderRes = result.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderRes = tsRawData.substring(181, 182); - BigInteger UDPIntMetaDataHeaderResBigInt = new BigInteger(UDPIntMetaDataHeaderRes, 16); - BigInteger bitwiseAnd = new BigInteger('C', 16); - BigInteger result = UDPIntMetaDataHeaderResBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderRes = result.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderD = tsRawData.substring(141, 142); - BigInteger UDPIntMetaDataHeaderDBigInt = new BigInteger(UDPIntMetaDataHeaderD, 16); - BigInteger bitwiseAnd = new BigInteger('2', 16); - BigInteger result = UDPIntMetaDataHeaderDBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderD = result.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderD = tsRawData.substring(181, 182); - BigInteger UDPIntMetaDataHeaderDBigInt = new BigInteger(UDPIntMetaDataHeaderD, 16); - BigInteger bitwiseAnd = new BigInteger('2', 16); - BigInteger result = UDPIntMetaDataHeaderDBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderD = result.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderE = tsRawData.substring(141, 142); - BigInteger UDPIntMetaDataHeaderEBigInt = new BigInteger(UDPIntMetaDataHeaderE, 16); - BigInteger bitwiseAnd = new BigInteger('1', 16); - BigInteger result = UDPIntMetaDataHeaderEBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderE = result.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderE = tsRawData.substring(181, 182); - BigInteger UDPIntMetaDataHeaderEBigInt = new BigInteger(UDPIntMetaDataHeaderE, 16); - BigInteger bitwiseAnd = new BigInteger('1', 16); - BigInteger result = UDPIntMetaDataHeaderEBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderE = result.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderM = tsRawData.substring(142, 143); - BigInteger UDPIntMetaDataHeaderMBigInt = new BigInteger(UDPIntMetaDataHeaderM, 16); - BigInteger bitwiseAnd = new BigInteger('8', 16); - BigInteger result = UDPIntMetaDataHeaderMBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderM = result.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderM = tsRawData.substring(182, 183); - BigInteger UDPIntMetaDataHeaderMBigInt = new BigInteger(UDPIntMetaDataHeaderM, 16); - BigInteger bitwiseAnd = new BigInteger('8', 16); - BigInteger result = UDPIntMetaDataHeaderMBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderM = result.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderPerhopMetadataLengthStr = tsRawData.substring(144, 146); - BigInteger UDPIntMetaDataHeaderPerhopMetadataLengthBigInt = new BigInteger(UDPIntMetaDataHeaderPerhopMetadataLengthStr, 16); - BigInteger bitwiseAnd = new BigInteger('1F', 16); - BigInteger result = UDPIntMetaDataHeaderPerhopMetadataLengthBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderPerhopMetadataLength = result.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderPerhopMetadataLengthStr = tsRawData.substring(184, 186); - BigInteger UDPIntMetaDataHeaderPerhopMetadataLengthBigInt = new BigInteger(UDPIntMetaDataHeaderPerhopMetadataLengthStr, 16); - BigInteger bitwiseAnd = new BigInteger('1F', 16); - BigInteger result = UDPIntMetaDataHeaderPerhopMetadataLengthBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderPerhopMetadataLength = result.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderRemainingHopCntStr = tsRawData.substring(146, 148); - BigInteger UDPIntMetaDataHeaderRemainingHopCntBigInt = new BigInteger(UDPIntMetaDataHeaderRemainingHopCntStr, 16); - ctx.ts.UDPIntMetaDataHeaderRemainingHopCnt = UDPIntMetaDataHeaderRemainingHopCntBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderRemainingHopCntStr = tsRawData.substring(186, 188); - BigInteger UDPIntMetaDataHeaderRemainingHopCntBigInt = new BigInteger(UDPIntMetaDataHeaderRemainingHopCntStr, 16); - ctx.ts.UDPIntMetaDataHeaderRemainingHopCnt = UDPIntMetaDataHeaderRemainingHopCntBigInt.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderInstructionBitmapStr = tsRawData.substring(148, 152); - BigInteger UDPIntMetaDataHeaderInstructionBitmapBigInt = new BigInteger(UDPIntMetaDataHeaderInstructionBitmapStr, 16); - ctx.ts.UDPIntMetaDataHeaderInstructionBitmap = UDPIntMetaDataHeaderInstructionBitmapBigInt.toString(16); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderInstructionBitmapStr = tsRawData.substring(188, 192); - BigInteger UDPIntMetaDataHeaderInstructionBitmapBigInt = new BigInteger(UDPIntMetaDataHeaderInstructionBitmapStr, 16); - ctx.ts.UDPIntMetaDataHeaderInstructionBitmap = UDPIntMetaDataHeaderInstructionBitmapBigInt.toString(16); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderDomainSpecificIDStr = tsRawData.substring(152, 156); - BigInteger UDPIntMetaDataHeaderDomainSpecificIDBigInt = new BigInteger(UDPIntMetaDataHeaderDomainSpecificIDStr, 16); - ctx.ts.UDPIntMetaDataHeaderDomainSpecificID = UDPIntMetaDataHeaderDomainSpecificIDBigInt.toString(16); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderDomainSpecificIDStr = tsRawData.substring(192, 196); - BigInteger UDPIntMetaDataHeaderDomainSpecificIDBigInt = new BigInteger(UDPIntMetaDataHeaderDomainSpecificIDStr, 16); - ctx.ts.UDPIntMetaDataHeaderDomainSpecificID = UDPIntMetaDataHeaderDomainSpecificIDBigInt.toString(16); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderDSInstructionStr = tsRawData.substring(156, 160); - BigInteger UDPIntMetaDataHeaderDSInstructionBigInt = new BigInteger(UDPIntMetaDataHeaderDSInstructionStr, 16); - ctx.ts.UDPIntMetaDataHeaderDSInstruction = UDPIntMetaDataHeaderDSInstructionBigInt.toString(16); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderDSInstructionStr = tsRawData.substring(196, 200); - BigInteger UDPIntMetaDataHeaderDSInstructionBigInt = new BigInteger(UDPIntMetaDataHeaderDSInstructionStr, 16); - ctx.ts.UDPIntMetaDataHeaderDSInstruction = UDPIntMetaDataHeaderDSInstructionBigInt.toString(16); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderDSFlagsStr = tsRawData.substring(160, 164); - BigInteger UDPIntMetaDataHeaderDSFlagsBigInt = new BigInteger(UDPIntMetaDataHeaderDSFlagsStr, 16); - ctx.ts.UDPIntMetaDataHeaderDSFlags = UDPIntMetaDataHeaderDSFlagsBigInt.toString(16); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderDSFlagsStr = tsRawData.substring(200, 204); - BigInteger UDPIntMetaDataHeaderDSFlagsBigInt = new BigInteger(UDPIntMetaDataHeaderDSFlagsStr, 16); - ctx.ts.UDPIntMetaDataHeaderDSFlags = UDPIntMetaDataHeaderDSFlagsBigInt.toString(16); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.INTMetadataStackData = tsRawData.substring(164, 196); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.INTMetadataStackData = tsRawData.substring(204, 236); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.INTMetadataStackSwitchID = tsRawData.substring(164, 172); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.INTMetadataStackSwitchID = tsRawData.substring(204, 212); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.INTMetadataSourceMetadata = tsRawData.substring(172, 196); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.INTMetadataSourceMetadata = tsRawData.substring(212, 236); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.INTMetadataSourceMetadataSwitchId = tsRawData.substring(172, 180); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.INTMetadataSourceMetadataSwitchId = tsRawData.substring(212, 220); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String rawOriginatingMac = tsRawData.substring(180, 192); - StringBuilder formattedOriginatingMac = new StringBuilder(); - for(int i = 0; i < rawOriginatingMac.length();) - { - formattedOriginatingMac.append(rawOriginatingMac.substring(i++, ++i)); - if(i < rawOriginatingMac.length()) formattedOriginatingMac.append(":"); - } - ctx.ts.INTMetadataSourceMetadataOriginatingMac = formattedOriginatingMac.toString(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String rawOriginatingMac = tsRawData.substring(220, 232); - StringBuilder formattedOriginatingMac = new StringBuilder(); - for(int i = 0; i < rawOriginatingMac.length();) - { - formattedOriginatingMac.append(rawOriginatingMac.substring(i++, ++i)); - if(i < rawOriginatingMac.length()) formattedOriginatingMac.append(":"); - } - ctx.ts.INTMetadataSourceMetadataOriginatingMac = formattedOriginatingMac.toString(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.INTMetadataSourceMetadataReserved = tsRawData.substring(192, 196); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.INTMetadataSourceMetadataReserved = tsRawData.substring(232, 236); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDP2Data = tsRawData.substring(196, 212); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDP2Data = tsRawData.substring(236, 252); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDP2SrcPortStr = tsRawData.substring(196, 200); - BigInteger UDP2SrcPortBigInt = new BigInteger(UDP2SrcPortStr,16); - ctx.ts.UDP2SrcPort = UDP2SrcPortBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDP2SrcPortStr = tsRawData.substring(236, 240); - BigInteger UDP2SrcPortBigInt = new BigInteger(UDP2SrcPortStr,16); - ctx.ts.UDP2SrcPort = UDP2SrcPortBigInt.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDP2DstPortStr = tsRawData.substring(200, 204); - BigInteger UDP2DstPortBigInt = new BigInteger(UDP2DstPortStr,16); - ctx.ts.UDP2DstPort = UDP2DstPortBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDP2DstPortStr = tsRawData.substring(240, 244); - BigInteger UDP2DstPortBigInt = new BigInteger(UDP2DstPortStr,16); - ctx.ts.UDP2DstPort = UDP2DstPortBigInt.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDP2Length = tsRawData.substring(204, 208); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDP2Length = tsRawData.substring(244, 248); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDP2Checksum = tsRawData.substring(208, 212); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDP2Checksum = tsRawData.substring(248, 252); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.dataII = tsRawData.substring(212, 252); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.dataII = tsRawData.substring(252, 292); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == true) - { - String dropTelemetryReport = ctx.ts.dropTelemetryReport; - String telemetryReportVarOpBsmdStr = dropTelemetryReport.substring(40,48); - BigInteger telemetryReportVarOpBsmdBigInt = new BigInteger(telemetryReportVarOpBsmdStr, 16); - ctx.ts.telemetryReportVarOpBsmd = telemetryReportVarOpBsmdBigInt.intValue(); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == true) - { - String dropTelemetryReport = ctx.ts.dropTelemetryReport; - String telemetryReportVarOptTimeStampStr = dropTelemetryReport.substring(48,56); - BigInteger telemetryReportVarOptTimeStampBigInt = new BigInteger(telemetryReportVarOptTimeStampStr, 16); - ctx.ts.telemetryReportVarOptTimeStamp = telemetryReportVarOptTimeStampBigInt.intValue(); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == true) - { - String dropTelemetryReport = ctx.ts.dropTelemetryReport; - String telemetryReportVarOptDropCountStr = dropTelemetryReport.substring(56,64); - BigInteger telemetryReportVarOptDropCountBigInt = new BigInteger(telemetryReportVarOptDropCountStr, 16); - ctx.ts.telemetryReportVarOptDropCount = telemetryReportVarOptDropCountBigInt.intValue(); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == true) - { - String dropTelemetryReport = ctx.ts.dropTelemetryReport; - String telemetryReportVarOptHashStr = dropTelemetryReport.substring(64,96); - BigInteger telemetryReportVarOptHashBigInt = new BigInteger(telemetryReportVarOptHashStr, 16); - ctx.ts.telemetryReportVarOptHash = telemetryReportVarOptHashBigInt; - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String macAddress = ctx.ts.INTMetadataSourceMetadataOriginatingMac; - String destinationPort = ctx.ts.UDP2DstPort.toString(); - String IPvDestIP = ctx.ts.IPv4DestIP; - String combinedKeys = ''; - if(ctx.ts.isIPv4 == true) - { - combinedKeys = combinedKeys.concat(macAddress).concat('|').concat(destinationPort).concat('|').concat(IPvDestIP).concat('|').concat('::'); - ctx.ts.dropHashInput = combinedKeys; - } - if(ctx.ts.isIPv6 == true) - { - combinedKeys = combinedKeys.concat(macAddress).concat('|').concat(destinationPort).concat('|').concat(IPvDestIP).concat('|').concat('::'); - ctx.ts.dropHashInput = combinedKeys; - } - } - """, - "ignore_failure" : false - } - }, - - { - "script" : { - "ignore_failure" : false, - "id" : "calculateKeyHash" - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String completeSHA256Hash = ctx.ts.sha256Hash; - String subsetSHA256HashStr= completeSHA256Hash.substring(0,16); - BigInteger hashBigInt = new BigInteger(subsetSHA256HashStr, 16); - ctx.ts.dropReportHash = hashBigInt; - """, - "ignore_failure" : true - } - } - ] - } diff --git a/playbooks/hcp/templates/udp/udp_monitor_def.json b/playbooks/hcp/templates/udp/udp_monitor_def.json deleted file mode 100644 index 5733cf77..00000000 --- a/playbooks/hcp/templates/udp/udp_monitor_def.json +++ /dev/null @@ -1,166 +0,0 @@ -{ - "type": "monitor", - "name": "DDOS-UDP-Monitor", - "enabled": true, - "schedule": { - "period": { - "interval": 1, - "unit": "MINUTES" - } - }, - "inputs": [{ - "search": { - "indices": ["packets-*"], - "query": { - "size": 0, - "query": { - "bool": { - "filter": [ - { - "range": { - "timestamp": { - "from": "{{period_end}}||-1m", - "to": "{{period_end}}", - "include_lower": true, - "include_upper": true, - "format": "epoch_millis", - "boost": 1 - } - } - }, - { - "wildcard": { - "ts.UDP2DstPort": { - "wildcard": "*5792*", - "boost": 1 - } - } - } - ], - "adjust_pure_negative": true, - "boost": 1 - } - }, - "_source": { - "includes": [ - "ts.UDP2DstPort", - "ts.IPvDestAddr", - "ts.INTMetadataSourceMetadataOriginatingMac", - "ts.IPv4SrcIP", - "ts.IPv4DestIP", - "ts.IPv6SrcIP", - "ts.IPv6DestIP" - ], - "excludes": [] - }, - "aggregations": { - "UDP2DstPortValueCount": { - "value_count": { - "field": "ts.UDP2DstPort" - } - }, - "IPvDestAddrValueCount": { - "value_count": { - "field": "ts.IPvDestAddr" - } - }, - "INTMetadataSourceMetadataOriginatingMacValueCount": { - "value_count": { - "field": "ts.INTMetadataSourceMetadataOriginatingMac" - } - }, - "OriginatingMacAddrUDPv4": { - "terms": { - "field": "ts.INTMetadataSourceMetadataOriginatingMac", - "size": 10, - "min_doc_count": 1, - "shard_min_doc_count": 0, - "show_term_doc_count_error": false, - "order": [ - { - "_count": "desc" - }, - { - "_key": "asc" - } - ] - } - }, - "IPv4SrcIP": { - "terms": { - "field": "ts.IPv4SrcIP", - "size": 10, - "min_doc_count": 1, - "shard_min_doc_count": 0, - "show_term_doc_count_error": false, - "order": [ - { - "_count": "desc" - }, - { - "_key": "asc" - } - ] - } - }, - "IPv4DestIP": { - "terms": { - "field": "ts.IPv4DestIP", - "size": 10, - "min_doc_count": 1, - "shard_min_doc_count": 0, - "show_term_doc_count_error": false, - "order": [ - { - "_count": "desc" - }, - { - "_key": "asc" - } - ] - } - }, - "UDP2DstPortValue": { - "terms": { - "field": "ts.UDP2DstPort", - "size": 10, - "min_doc_count": 1, - "shard_min_doc_count": 0, - "show_term_doc_count_error": false, - "order": [ - { - "_count": "desc" - }, - { - "_key": "asc" - } - ] - } - } - } - } - } - }], - "triggers": [{ - "name": "DDOS-Trigger", - "severity": "1", - "condition": { - "script": { - "source": "if(ctx.results[0].hits.total.value > 0){ctx.results[0].aggregations.UDP2DstPortValueCount.value >= 10}", - "lang": "painless" - } - }, - "actions": [{ - "name": "Sdn-WebHook-Action", - "destination_id": "sdnWebhookResponseId", - "message_template": { - "source":"{\"event_action\":\"trigger\", \"dst_ip\":\"{{ctx.results.0.aggregations.IPv4DestIP.buckets.0.key}}\",\"dst_port\":\"{{ctx.results.0.aggregations.UDP2DstPortValue.buckets.0.key}}\",\"src_ip\":\"{{ctx.results.0.aggregations.IPv4SrcIP.buckets.0.key}}\",\"src_mac\":\"{{ctx.results.0.aggregations.OriginatingMacAddrUDPv4.buckets.0.key}}\"}", - "lang": "mustache", - "options": { - "content_type": "application/json" - } - }, - "throttle_enabled": false - }] - }] -} diff --git a/playbooks/hcp/templates/udp_and_tcp_data_mapping.json b/playbooks/hcp/templates/udp_and_tcp_data_mapping.json deleted file mode 100644 index 7bc5bc80..00000000 --- a/playbooks/hcp/templates/udp_and_tcp_data_mapping.json +++ /dev/null @@ -1,320 +0,0 @@ -{ - "index_patterns" : [ - "packets-*" - ], - "settings" : { - "index" : { - "number_of_replicas" : "0", - "default_pipeline" : "ts_parsing", - "max_script_fields" : "200" - } - }, - "mappings" : { - "dynamic" : "false", - "properties" : { - "layers" : { - "properties" : { - "data_raw" : { - "type" : "keyword" - }, - "data" : { - "properties" : { - "data_data_data_raw" : { - "type" : "keyword" - }, - "data_data_len" : { - "type" : "keyword" - }, - "data_data_data" : { - "type" : "keyword" - } - } - } - } - }, - "timestamp" : { - "type" : "date" - }, - "ts" : { - "properties" : { - "INTMetadataSourceMetadataReserved" : { - "type" : "keyword" - }, - "UDPIntShimHeaderType" : { - "type" : "keyword" - }, - "telemetryReportLength" : { - "type" : "keyword" - }, - "data" : { - "type" : "keyword" - }, - "ethernetHeaderDestMac" : { - "type" : "keyword" - }, - "telemetryRsvd" : { - "type" : "keyword" - }, - "UDGSrcPort" : { - "type" : "keyword" - }, - "INTMetadataSourceMetadataSwitchId" : { - "type" : "keyword" - }, - "ethernetHeaderSrcMac" : { - "type" : "keyword" - }, - "UDPIntShimHeaderNextProto" : { - "type" : "keyword" - }, - "isIPv4" : { - "type" : "keyword" - }, - "isTCP" : { - "type" : "keyword" - }, - "isTCPv4" : { - "type" : "keyword" - }, - "isTCPv6" : { - "type" : "keyword" - }, - "isUDP" : { - "type" : "keyword" - }, - "isUDPv4" : { - "type" : "keyword" - }, - "isUDPv6" : { - "type" : "keyword" - }, - "dropReportHash": { - "type": "keyword" - }, - "sha256Hash": { - "type": "keyword" - }, - "nextProto": { - "type": "keyword" - }, - "TCPData" : { - "type" : "keyword" - }, - "TCPSrcPort" : { - "type" : "keyword" - }, - "TCPDestPort" : { - "type" : "keyword" - }, - "TCPIntShimHeaderNextProto" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderVersion" : { - "type" : "keyword" - }, - "IPvNextHdrProto" : { - "type" : "keyword" - }, - "telemetryReportVersion" : { - "type" : "keyword" - }, - "isIPv6" : { - "type" : "keyword" - }, - "IPv4SrcIP": { - "type" : "keyword" - }, - "UDP2DstPort": { - "type": "keyword" - }, - "UDP2Checksum": { - "type": "keyword" - }, - "UDP2Length": { - "type": "keyword" - }, - "combinedHashKeys": { - "type": "keyword" - }, - "UDP2SrcPort": { - "type": "keyword" - }, - "IPv6SrcIP": { - "type" : "keyword" - }, - "IPv4DestIP":{ - "type" : "keyword" - }, - "IPv6DestIP":{ - "type" : "keyword" - }, - "INTMetadataSourceMetadataOriginatingMac" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderInstructionBitmap" : { - "type" : "keyword" - }, - "INTMetadataSourceMetadata" : { - "type" : "keyword" - }, - "IPvSrcAddr" : { - "type" : "keyword" - }, - "tsRawDataSize" : { - "type" : "keyword" - }, - "UDGPData" : { - "type" : "keyword" - }, - "UDGChecksum" : { - "type" : "keyword" - }, - "INTMetadataHeaderData" : { - "type" : "keyword" - }, - "IPvDestAddr" : { - "type" : "keyword" - }, - "telemetryInType" : { - "type" : "keyword" - }, - "telemetryReport" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderDSFlags" : { - "type" : "keyword" - }, - "telemetryDsMdStatus" : { - "type" : "keyword" - }, - "InBandNetworkTelemetryData" : { - "type" : "keyword" - }, - "UDPIntShimHeaderRes2" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderDSInstruction" : { - "type" : "keyword" - }, - "UDPIntShimHeaderRes1" : { - "type" : "keyword" - }, - "UDP2Data" : { - "type" : "keyword" - }, - "telemetryHwdId" : { - "type" : "keyword" - }, - "telemetryVarOpt" : { - "type" : "keyword" - }, - "IPvData" : { - "type" : "keyword" - }, - "ethernetHeaderEtherType" : { - "type" : "keyword" - }, - "INTMetadataStackData" : { - "type" : "keyword" - }, - "IPvFlow" : { - "type" : "keyword" - }, - "UDPIntShimHeaderData" : { - "type" : "keyword" - }, - "INTMetadataStackSwitchID" : { - "type" : "keyword" - }, - "IPVersion" : { - "type" : "keyword" - }, - "IPvPayloadLength" : { - "type" : "keyword" - }, - "UDPIntShimHeaderLength" : { - "type" : "keyword" - }, - "UDPIntShimHeaderNPT" : { - "type" : "keyword" - }, - "UDGLength" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderM" : { - "type" : "keyword" - }, - "telemetryD" : { - "type" : "keyword" - }, - "rawData" : { - "type" : "keyword" - }, - "telemetryF" : { - "type" : "keyword" - }, - "telemetryReportType" : { - "type" : "keyword" - }, - "telemetryI" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderPerhopMetadataLength" : { - "type" : "keyword" - }, - "telemetryQ" : { - "type" : "keyword" - }, - "telemetryDomainSpecificID" : { - "type" : "keyword" - }, - "udpLayersSize" : { - "type" : "keyword" - }, - "UDGDstPort" : { - "type" : "keyword" - }, - "IPvNextHopLimit" : { - "type" : "keyword" - }, - "telemetryDsMdBits" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderDomainSpecificID" : { - "type" : "keyword" - }, - "telemetryRepMdBits" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderRemainingHopCnt" : { - "type" : "keyword" - }, - "telemetrySequenceNumber" : { - "type" : "keyword" - }, - "telemetryMDLength" : { - "type" : "keyword" - }, - "IPvClass" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderD" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderE" : { - "type" : "keyword" - }, - "telemetryNodeId" : { - "type" : "keyword" - }, - "dataII" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderRes" : { - "type" : "keyword" - } - } - } - } - } -} diff --git a/playbooks/hcp/templates/udp_and_tcp_data_parsing.json b/playbooks/hcp/templates/udp_and_tcp_data_parsing.json deleted file mode 100644 index e1ff806e..00000000 --- a/playbooks/hcp/templates/udp_and_tcp_data_parsing.json +++ /dev/null @@ -1,1777 +0,0 @@ -{ - "description" : "Transparent Security TCP & UDP v4 & v6 data parsing pipeline", - "processors" : [ - { - "set" : { - "field" : "ts.rawData", - "value" : "{{layers.data.data_data_data}}" - } - }, - { - "script" : { - "lang" : "painless", - "source" : "ctx.ts.telemetryReport =ctx.ts.rawData.substring(0, 48).replace(':','')", - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : "ctx.ts.dropTelemetryReport = ctx.ts.rawData.substring(0, 180).replace(':','')", - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String hardwareIdStr = ctx.layers.data.data_data_data.replace(':','').substring(1, 3); - BigInteger hwdIdBigInt = new BigInteger(hardwareIdStr, 16); - BigInteger bitwiseAnd = new BigInteger('FC', 16); - BigInteger extractedHardwareId = hwdIdBigInt.and(bitwiseAnd); - String hwdIdStr = extractedHardwareId.toString(16); - ctx.ts.telemetryHwdId = hwdIdStr; - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String sequenceNumberPartStrI = tsRawData.substring(2, 3); - String sequenceNumberPartStrII = tsRawData.substring(3, 8); - String sequenceNumberPartStr = tsRawData.substring(2, 8); - BigInteger sequenceNumberBigInt = new BigInteger(sequenceNumberPartStr, 16); - BigInteger bitwiseAnd = new BigInteger('7FFFFFF', 16); - BigInteger updatedSequenceNumber = sequenceNumberBigInt.and(bitwiseAnd); - // String sequenceNumberHexStr = updatedSequenceNumber.toString(16); - ctx.ts.telemetrySequenceNumber = updatedSequenceNumber.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String nodeIdStr = tsRawData.substring(8, 16); - BigInteger nodeIdBigInt = new BigInteger(nodeIdStr, 16); - ctx.ts.telemetryNodeId =nodeIdBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String repTypeStr = tsRawData.substring(16, 17); - BigInteger repTypeBigInt = new BigInteger(repTypeStr, 16); - ctx.ts.telemetryReportType =repTypeBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String inTypeStr = tsRawData.substring(17, 18); - BigInteger inTypeBigInt = new BigInteger(inTypeStr, 16); - ctx.ts.telemetryInType = inTypeBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String reportLengthStr = tsRawData.substring(18, 20); - BigInteger reportLengthBigInt = new BigInteger(reportLengthStr, 16); - ctx.ts.telemetryReportLength = reportLengthBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String mdLengthStr = tsRawData.substring(20, 22); - BigInteger mdLengthBigInt = new BigInteger(mdLengthStr, 16); - ctx.ts.telemetryMDLength = mdLengthBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String dStr = tsRawData.substring(22, 23); - BigInteger dBigInt = new BigInteger(dStr, 16); - BigInteger bitwiseAnd = new BigInteger('8', 16); - BigInteger dRes = dBigInt.and(bitwiseAnd); - ctx.ts.telemetryD = dRes.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String qStr = tsRawData.substring(22, 23); - BigInteger qBigInt = new BigInteger(qStr, 16); - BigInteger bitwiseAnd = new BigInteger('4', 16); - BigInteger qRes = qBigInt.and(bitwiseAnd); - ctx.ts.telemetryQ = qRes.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String fStr = tsRawData.substring(22, 23); - BigInteger fBigInt = new BigInteger(fStr, 16); - BigInteger bitwiseAnd = new BigInteger('2', 16); - BigInteger fRes = fBigInt.and(bitwiseAnd); - ctx.ts.telemetryF = fRes.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String IStr = tsRawData.substring(22, 23); - BigInteger IBigInt = new BigInteger(IStr, 16); - BigInteger bitwiseAnd = new BigInteger('1', 16); - BigInteger IRes = IBigInt.and(bitwiseAnd); - ctx.ts.telemetryI = IRes.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String rsvdStr = tsRawData.substring(23, 24); - BigInteger rsvdBigInt = new BigInteger(rsvdStr, 16); - ctx.ts.telemetryRsvd = rsvdBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String repMdBitsStr = tsRawData.substring(24, 28); - BigInteger repMdBitsBigInt = new BigInteger(repMdBitsStr, 16); - ctx.ts.telemetryRepMdBits = repMdBitsBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String domainSpecificIDStr = tsRawData.substring(28, 32); - BigInteger domainSpecificIDBigInt = new BigInteger(domainSpecificIDStr, 16); - ctx.ts.telemetryDomainSpecificID = domainSpecificIDBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String dsMdBitsStr = tsRawData.substring(32, 36); - BigInteger dsMdBitsStrBigInt = new BigInteger(dsMdBitsStr, 16); - ctx.ts.telemetryDsMdBits = dsMdBitsStrBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String dsMdStatusStr = tsRawData.substring(36, 40); - BigInteger dsMdStatusBigInt = new BigInteger(dsMdStatusStr, 16); - ctx.ts.telemetryDsMdStatus = dsMdStatusBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String varOptStr = tsRawData.substring(40, 48); - BigInteger varOptBigInt = new BigInteger(varOptStr, 16); - ctx.ts.telemetryVarOpt = varOptBigInt.intValue(); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.isIPv6 = true; - } else - { - ctx.ts.isIPv6 = false; - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.isIPv4 = true; - } else - { - ctx.ts.isIPv4 = false; - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - String dropReportStrValue = '2'; - String telemetryIPv4StrValue ='4'; - String telemetryIPv6StrValue ='5'; - String telemetryInTypeStrValue = ctx.ts.telemetryInType.toString(); - if(dropReportStrValue == telemetryInTypeStrValue) - { - ctx.ts.isDropReport = true; - } - if( (telemetryInTypeStrValue == telemetryIPv4StrValue) || (telemetryInTypeStrValue == telemetryIPv6StrValue) ) - { - ctx.ts.isDropReport = false; - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String nextProtoStr = tsRawData.substring(138, 140); - BigInteger nextProtoBigInt = new BigInteger(nextProtoStr, 16); - ctx.ts.nextProto = nextProtoBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String nextProtoStr = tsRawData.substring(178, 180); - BigInteger nextProtoBigInt = new BigInteger(nextProtoStr, 16); - ctx.ts.nextProto = nextProtoBigInt.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - String tcpNextProtoStr = '6'; - int tcpNextProtoInt = Integer.parseInt(tcpNextProtoStr); - - if( etherType == '0800' && ipVersion == '4') - { - - if( ctx.ts.nextProto == tcpNextProtoInt ) - { - ctx.ts.isTCPv4 = true; - }else { - ctx.ts.isTCPv4 = false; - } - } - - if( etherType == '86dd' && ipVersion == '6') - { - - if( ctx.ts.nextProto == tcpNextProtoInt ) - { - ctx.ts.isTCPv6 = true; - }else { - ctx.ts.isTCPv6 = false; - } - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - String udpNextProtoStr = '17'; - int udpNextProtoInt = Integer.parseInt(udpNextProtoStr); - if( etherType == '0800' && ipVersion == '4') - { - - if(ctx.ts.nextProto == udpNextProtoInt ) - { - ctx.ts.isUDPv4 = true; - }else { - ctx.ts.isUDPv4 = false; - } - } - - if( etherType == '86dd' && ipVersion == '6') - { - String nextProtoStr = tsRawData.substring(178, 180); - BigInteger nextProtoBigInt = new BigInteger(nextProtoStr, 16); - - if(ctx.ts.nextProto == udpNextProtoInt ) - { - ctx.ts.isUDPv6 = true; - }else { - ctx.ts.isUDPv6 = false; - } - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.data = tsRawData.substring(48, 252); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.data = tsRawData.substring(48, 292); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - ctx.ts.ethernetHeader = tsRawData.substring(48, 76); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - ctx.ts.ethernetHeaderDestMac = tsRawData.substring(48, 60); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - ctx.ts.ethernetHeaderSrcMac = tsRawData.substring(60, 72); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - ctx.ts.ethernetHeaderEtherType = tsRawData.substring(72, 76); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.IPvData = tsRawData.substring(76, 116); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.IPvData = tsRawData.substring(76, 156); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.IPVersion = tsRawData.substring(76, 77); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.IPVersion = tsRawData.substring(76, 77); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String IPv4SrcIPHexString = tsRawData.substring(100, 108); - String IPv4SrcIP = ""; - for(int i = 0; i < IPv4SrcIPHexString.length(); i = i + 2) { - IPv4SrcIP = IPv4SrcIP + Integer.valueOf(IPv4SrcIPHexString.substring(i, i+2), 16) + "."; - } - IPv4SrcIP = IPv4SrcIP.substring(0, IPv4SrcIP.length()-1); - ctx.ts.IPv4SrcIP = IPv4SrcIP; - } - if( etherType == '86dd' && ipVersion == '6') - { - String IPv6SrcIPHexString = tsRawData.substring(92,124); - String IPv6SrcIP = ""; - for(int i = 0; i < IPv6SrcIPHexString.length(); i = i + 4) { - IPv6SrcIP = IPv6SrcIP + IPv6SrcIPHexString.substring(i, i+4) + ":"; - } - IPv6SrcIP = IPv6SrcIP.substring(0, IPv6SrcIP.length()-1); - ctx.ts.IPv6SrcIP = IPv6SrcIP; - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String IPv4DestIPHexString = tsRawData.substring(108, 116); - String IPv4DestIP = ""; - for(int i = 0; i < IPv4DestIPHexString.length(); i = i + 2) { - IPv4DestIP = IPv4DestIP + Integer.valueOf(IPv4DestIPHexString.substring(i, i+2), 16) + "."; - } - IPv4DestIP = IPv4DestIP.substring(0, IPv4DestIP.length()-1); - ctx.ts.IPv4DestIP = IPv4DestIP; - } - if( etherType == '86dd' && ipVersion == '6') - { - String IPv6DestIPHexString = tsRawData.substring(124,156); - String IPv6DestIP = ""; - for(int i = 0; i < IPv6DestIPHexString.length(); i = i + 4) { - IPv6DestIP = IPv6DestIP + IPv6DestIPHexString.substring(i, i+4) + ":"; - } - IPv6DestIP = IPv6DestIP.substring(0, IPv6DestIP.length()-1); - ctx.ts.IPv6DestIP = IPv6DestIP; - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDGPData = tsRawData.substring(116, 132); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDGPData = tsRawData.substring(156, 172); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDGSrcPortStr = tsRawData.substring(116, 120); - BigInteger UDGSrcPortBigInt = new BigInteger(UDGSrcPortStr, 16); - ctx.ts.UDGSrcPort = UDGSrcPortBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDGSrcPortStr = tsRawData.substring(156, 160); - BigInteger UDGSrcPortBigInt = new BigInteger(UDGSrcPortStr, 16); - ctx.ts.UDGSrcPort = UDGSrcPortBigInt.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDGDstPortStr = tsRawData.substring(120, 124); - BigInteger UDGDstPortBigInt = new BigInteger(UDGDstPortStr, 16); - ctx.ts.UDGDstPort = UDGDstPortBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDGDstPortStr = tsRawData.substring(160, 164); - BigInteger UDGDstPortBigInt = new BigInteger(UDGDstPortStr, 16); - ctx.ts.UDGDstPort = UDGDstPortBigInt.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDGLengthStr = tsRawData.substring(124, 128); - BigInteger UDGLengthBigInt = new BigInteger(UDGLengthStr, 16); - ctx.ts.UDGLength = UDGLengthBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDGLengthStr = tsRawData.substring(164, 168); - BigInteger UDGLengthBigInt = new BigInteger(UDGLengthStr, 16); - ctx.ts.UDGLength = UDGLengthBigInt.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDGChecksumStr = tsRawData.substring(128, 132); - BigInteger UDGChecksumBigInt = new BigInteger(UDGChecksumStr, 16); - ctx.ts.UDGChecksum = UDGChecksumBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDGChecksumStr = tsRawData.substring(168, 172); - BigInteger UDGChecksumBigInt = new BigInteger(UDGChecksumStr, 16); - ctx.ts.UDGChecksum = UDGChecksumBigInt.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.InBandNetworkTelemetryData = tsRawData.substring(132, 196); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.InBandNetworkTelemetryData = tsRawData.substring(172, 236); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDPIntShimHeaderData = tsRawData.substring(132, 140); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDPIntShimHeaderData = tsRawData.substring(172, 180); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDPIntShimHeaderType = tsRawData.substring(132, 133); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDPIntShimHeaderType = tsRawData.substring(172, 173); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String udpIntShimNPTStr = tsRawData.substring(133, 134); - BigInteger udpIntShimNPTBigInt = new BigInteger(udpIntShimNPTStr, 16); - BigInteger bitwiseAnd = new BigInteger('C', 16); - BigInteger result = udpIntShimNPTBigInt.and(bitwiseAnd); - ctx.ts.UDPIntShimHeaderNPT = result.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String udpIntShimNPTStr = tsRawData.substring(173, 174); - BigInteger udpIntShimNPTBigInt = new BigInteger(udpIntShimNPTStr, 16); - BigInteger bitwiseAnd = new BigInteger('C', 16); - BigInteger result = udpIntShimNPTBigInt.and(bitwiseAnd); - ctx.ts.UDPIntShimHeaderNPT = result.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDPIntShimHeaderRes1 = tsRawData.substring(133, 134); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDPIntShimHeaderRes1 = tsRawData.substring(174, 175); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String lengthStr = tsRawData.substring(134, 136); - BigInteger lengthStrBigInt = new BigInteger(lengthStr, 16); - ctx.ts.UDPIntShimHeaderLength= lengthStrBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String lengthStr = tsRawData.substring(174, 176); - BigInteger lengthStrBigInt = new BigInteger(lengthStr, 16); - ctx.ts.UDPIntShimHeaderLength = lengthStrBigInt.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDPIntShimHeaderRes2 = tsRawData.substring(136, 138); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDPIntShimHeaderRes2 = tsRawData.substring(176, 178); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String nextProtoStr = tsRawData.substring(138, 140); - BigInteger nextProtoBigInt = new BigInteger(nextProtoStr, 16); - ctx.ts.UDPIntShimHeaderNextProto = nextProtoBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String nextProtoStr = tsRawData.substring(178, 180); - BigInteger nextProtoBigInt = new BigInteger(nextProtoStr, 16); - ctx.ts.UDPIntShimHeaderNextProto = nextProtoBigInt.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.INTMetadataHeaderData = tsRawData.substring(140, 164); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.INTMetadataHeaderData = tsRawData.substring(180, 204); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderVersionStr = tsRawData.substring(140, 141); - BigInteger UDPIntMetaDataHeaderVersionStrBigInt = new BigInteger(UDPIntMetaDataHeaderVersionStr, 16); - ctx.ts.UDPIntMetaDataHeaderVersion = UDPIntMetaDataHeaderVersionStrBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderVersionStr = tsRawData.substring(180, 181); - BigInteger UDPIntMetaDataHeaderVersionStrBigInt = new BigInteger(UDPIntMetaDataHeaderVersionStr, 16); - ctx.ts.UDPIntMetaDataHeaderVersion = UDPIntMetaDataHeaderVersionStrBigInt.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderRes = tsRawData.substring(141, 142); - BigInteger UDPIntMetaDataHeaderResBigInt = new BigInteger(UDPIntMetaDataHeaderRes, 16); - BigInteger bitwiseAnd = new BigInteger('C', 16); - BigInteger result = UDPIntMetaDataHeaderResBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderRes = result.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderRes = tsRawData.substring(181, 182); - BigInteger UDPIntMetaDataHeaderResBigInt = new BigInteger(UDPIntMetaDataHeaderRes, 16); - BigInteger bitwiseAnd = new BigInteger('C', 16); - BigInteger result = UDPIntMetaDataHeaderResBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderRes = result.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderD = tsRawData.substring(141, 142); - BigInteger UDPIntMetaDataHeaderDBigInt = new BigInteger(UDPIntMetaDataHeaderD, 16); - BigInteger bitwiseAnd = new BigInteger('2', 16); - BigInteger result = UDPIntMetaDataHeaderDBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderD = result.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderD = tsRawData.substring(181, 182); - BigInteger UDPIntMetaDataHeaderDBigInt = new BigInteger(UDPIntMetaDataHeaderD, 16); - BigInteger bitwiseAnd = new BigInteger('2', 16); - BigInteger result = UDPIntMetaDataHeaderDBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderD = result.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderE = tsRawData.substring(141, 142); - BigInteger UDPIntMetaDataHeaderEBigInt = new BigInteger(UDPIntMetaDataHeaderE, 16); - BigInteger bitwiseAnd = new BigInteger('1', 16); - BigInteger result = UDPIntMetaDataHeaderEBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderE = result.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderE = tsRawData.substring(181, 182); - BigInteger UDPIntMetaDataHeaderEBigInt = new BigInteger(UDPIntMetaDataHeaderE, 16); - BigInteger bitwiseAnd = new BigInteger('1', 16); - BigInteger result = UDPIntMetaDataHeaderEBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderE = result.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderM = tsRawData.substring(142, 143); - BigInteger UDPIntMetaDataHeaderMBigInt = new BigInteger(UDPIntMetaDataHeaderM, 16); - BigInteger bitwiseAnd = new BigInteger('8', 16); - BigInteger result = UDPIntMetaDataHeaderMBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderM = result.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderM = tsRawData.substring(182, 183); - BigInteger UDPIntMetaDataHeaderMBigInt = new BigInteger(UDPIntMetaDataHeaderM, 16); - BigInteger bitwiseAnd = new BigInteger('8', 16); - BigInteger result = UDPIntMetaDataHeaderMBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderM = result.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderPerhopMetadataLengthStr = tsRawData.substring(144, 146); - BigInteger UDPIntMetaDataHeaderPerhopMetadataLengthBigInt = new BigInteger(UDPIntMetaDataHeaderPerhopMetadataLengthStr, 16); - BigInteger bitwiseAnd = new BigInteger('1F', 16); - BigInteger result = UDPIntMetaDataHeaderPerhopMetadataLengthBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderPerhopMetadataLength = result.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderPerhopMetadataLengthStr = tsRawData.substring(184, 186); - BigInteger UDPIntMetaDataHeaderPerhopMetadataLengthBigInt = new BigInteger(UDPIntMetaDataHeaderPerhopMetadataLengthStr, 16); - BigInteger bitwiseAnd = new BigInteger('1F', 16); - BigInteger result = UDPIntMetaDataHeaderPerhopMetadataLengthBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderPerhopMetadataLength = result.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderRemainingHopCntStr = tsRawData.substring(146, 148); - BigInteger UDPIntMetaDataHeaderRemainingHopCntBigInt = new BigInteger(UDPIntMetaDataHeaderRemainingHopCntStr, 16); - ctx.ts.UDPIntMetaDataHeaderRemainingHopCnt = UDPIntMetaDataHeaderRemainingHopCntBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderRemainingHopCntStr = tsRawData.substring(186, 188); - BigInteger UDPIntMetaDataHeaderRemainingHopCntBigInt = new BigInteger(UDPIntMetaDataHeaderRemainingHopCntStr, 16); - ctx.ts.UDPIntMetaDataHeaderRemainingHopCnt = UDPIntMetaDataHeaderRemainingHopCntBigInt.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderInstructionBitmapStr = tsRawData.substring(148, 152); - BigInteger UDPIntMetaDataHeaderInstructionBitmapBigInt = new BigInteger(UDPIntMetaDataHeaderInstructionBitmapStr, 16); - ctx.ts.UDPIntMetaDataHeaderInstructionBitmap = UDPIntMetaDataHeaderInstructionBitmapBigInt.toString(16); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderInstructionBitmapStr = tsRawData.substring(188, 192); - BigInteger UDPIntMetaDataHeaderInstructionBitmapBigInt = new BigInteger(UDPIntMetaDataHeaderInstructionBitmapStr, 16); - ctx.ts.UDPIntMetaDataHeaderInstructionBitmap = UDPIntMetaDataHeaderInstructionBitmapBigInt.toString(16); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderDomainSpecificIDStr = tsRawData.substring(152, 156); - BigInteger UDPIntMetaDataHeaderDomainSpecificIDBigInt = new BigInteger(UDPIntMetaDataHeaderDomainSpecificIDStr, 16); - ctx.ts.UDPIntMetaDataHeaderDomainSpecificID = UDPIntMetaDataHeaderDomainSpecificIDBigInt.toString(16); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderDomainSpecificIDStr = tsRawData.substring(192, 196); - BigInteger UDPIntMetaDataHeaderDomainSpecificIDBigInt = new BigInteger(UDPIntMetaDataHeaderDomainSpecificIDStr, 16); - ctx.ts.UDPIntMetaDataHeaderDomainSpecificID = UDPIntMetaDataHeaderDomainSpecificIDBigInt.toString(16); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderDSInstructionStr = tsRawData.substring(156, 160); - BigInteger UDPIntMetaDataHeaderDSInstructionBigInt = new BigInteger(UDPIntMetaDataHeaderDSInstructionStr, 16); - ctx.ts.UDPIntMetaDataHeaderDSInstruction = UDPIntMetaDataHeaderDSInstructionBigInt.toString(16); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderDSInstructionStr = tsRawData.substring(196, 200); - BigInteger UDPIntMetaDataHeaderDSInstructionBigInt = new BigInteger(UDPIntMetaDataHeaderDSInstructionStr, 16); - ctx.ts.UDPIntMetaDataHeaderDSInstruction = UDPIntMetaDataHeaderDSInstructionBigInt.toString(16); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderDSFlagsStr = tsRawData.substring(160, 164); - BigInteger UDPIntMetaDataHeaderDSFlagsBigInt = new BigInteger(UDPIntMetaDataHeaderDSFlagsStr, 16); - ctx.ts.UDPIntMetaDataHeaderDSFlags = UDPIntMetaDataHeaderDSFlagsBigInt.toString(16); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderDSFlagsStr = tsRawData.substring(200, 204); - BigInteger UDPIntMetaDataHeaderDSFlagsBigInt = new BigInteger(UDPIntMetaDataHeaderDSFlagsStr, 16); - ctx.ts.UDPIntMetaDataHeaderDSFlags = UDPIntMetaDataHeaderDSFlagsBigInt.toString(16); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.INTMetadataStackData = tsRawData.substring(164, 196); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.INTMetadataStackData = tsRawData.substring(204, 236); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.INTMetadataStackSwitchID = tsRawData.substring(164, 172); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.INTMetadataStackSwitchID = tsRawData.substring(204, 212); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.INTMetadataSourceMetadata = tsRawData.substring(172, 196); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.INTMetadataSourceMetadata = tsRawData.substring(212, 236); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.INTMetadataSourceMetadataSwitchId = tsRawData.substring(172, 180); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.INTMetadataSourceMetadataSwitchId = tsRawData.substring(212, 220); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String rawOriginatingMac = tsRawData.substring(180, 192); - StringBuilder formattedOriginatingMac = new StringBuilder(); - for(int i = 0; i < rawOriginatingMac.length();) - { - formattedOriginatingMac.append(rawOriginatingMac.substring(i++, ++i)); - if(i < rawOriginatingMac.length()) formattedOriginatingMac.append(":"); - } - ctx.ts.INTMetadataSourceMetadataOriginatingMac = formattedOriginatingMac.toString(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String rawOriginatingMac = tsRawData.substring(220, 232); - StringBuilder formattedOriginatingMac = new StringBuilder(); - for(int i = 0; i < rawOriginatingMac.length();) - { - formattedOriginatingMac.append(rawOriginatingMac.substring(i++, ++i)); - if(i < rawOriginatingMac.length()) formattedOriginatingMac.append(":"); - } - ctx.ts.INTMetadataSourceMetadataOriginatingMac = formattedOriginatingMac.toString(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.INTMetadataSourceMetadataReserved = tsRawData.substring(192, 196); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.INTMetadataSourceMetadataReserved = tsRawData.substring(232, 236); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if(ctx.ts.isUDPv4 == true || ctx.ts.isUDPv6 == true) - { - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDP2Data = tsRawData.substring(196, 212); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDP2Data = tsRawData.substring(236, 252); - } - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - String tcpNextProtoStr = '6'; - int tcpNextProtoInt = Integer.parseInt(tcpNextProtoStr); - if(ctx.ts.isTCPv4 == true || ctx.ts.isTCPv6 == true) - { - if( etherType == '0800' && ipVersion == '4' && ctx.ts.nextProto == tcpNextProtoInt ) - { - ctx.ts.TCPData = tsRawData.substring(196, 236); - } - if(etherType == '86dd' && ipVersion == '6' && ctx.ts.nextProto == tcpNextProtoInt ) - { - ctx.ts.TCPData = tsRawData.substring(236, 276); - } - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( ctx.ts.isUDPv4 == true || ctx.ts.isUDPv6 == true ) - { - if( etherType == '0800' && ipVersion == '4') - { - String UDP2SrcPortStr = tsRawData.substring(196, 200); - BigInteger UDP2SrcPortBigInt = new BigInteger(UDP2SrcPortStr,16); - ctx.ts.UDP2SrcPort = UDP2SrcPortBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDP2SrcPortStr = tsRawData.substring(236, 240); - BigInteger UDP2SrcPortBigInt = new BigInteger(UDP2SrcPortStr,16); - ctx.ts.UDP2SrcPort = UDP2SrcPortBigInt.intValue(); - } - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - String tcpNextProtoStr = '6'; - if( ctx.ts.isTCPv4 == true || ctx.ts.isTCPv6 == true ) - { - int tcpNextProtoInt = Integer.parseInt(tcpNextProtoStr); - if( etherType == '0800' && ipVersion == '4' ) - { - String TCPSrcPortStr = tsRawData.substring(196, 200); - BigInteger TCPSrcPortBigInt = new BigInteger(TCPSrcPortStr,16); - ctx.ts.TCPSrcPort = TCPSrcPortBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String TCPSrcPortStr = tsRawData.substring(236, 240); - BigInteger TCPSrcPortBigInt = new BigInteger(TCPSrcPortStr,16); - ctx.ts.TCPSrcPort = TCPSrcPortBigInt.intValue(); - } - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( ctx.ts.isUDPv4 == true || ctx.ts.isUDPv6 == true ) - { - if( etherType == '0800' && ipVersion == '4') - { - String UDP2DstPortStr = tsRawData.substring(200, 204); - BigInteger UDP2DstPortBigInt = new BigInteger(UDP2DstPortStr,16); - ctx.ts.UDP2DstPort = UDP2DstPortBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDP2DstPortStr = tsRawData.substring(240, 244); - BigInteger UDP2DstPortBigInt = new BigInteger(UDP2DstPortStr,16); - ctx.ts.UDP2DstPort = UDP2DstPortBigInt.intValue(); - } - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - String tcpNextProtoStr = '6'; - if( ctx.ts.isTCPv4 == true || ctx.ts.isTCPv6 == true ) - { - int tcpNextProtoInt = Integer.parseInt(tcpNextProtoStr); - if( etherType == '0800' && ipVersion == '4' && ctx.ts.nextProto == tcpNextProtoInt ) - { - String TCPDstPortStr = tsRawData.substring(200, 204); - BigInteger TCPDstPortBigInt = new BigInteger(TCPDstPortStr,16); - ctx.ts.TCPDestPort = TCPDstPortBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6' && ctx.ts.nextProto == tcpNextProtoInt ) - { - String TCPDstPortStr = tsRawData.substring(240, 244); - BigInteger TCPDstPortBigInt = new BigInteger(TCPDstPortStr,16); - ctx.ts.TCPDestPort = TCPDstPortBigInt.intValue(); - } - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDP2Length = tsRawData.substring(204, 208); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDP2Length = tsRawData.substring(244, 248); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == true) - { - String dropTelemetryReport = ctx.ts.dropTelemetryReport; - String telemetryReportVarOpBsmdStr = dropTelemetryReport.substring(40,48); - BigInteger telemetryReportVarOpBsmdBigInt = new BigInteger(telemetryReportVarOpBsmdStr, 16); - ctx.ts.telemetryReportVarOpBsmd = telemetryReportVarOpBsmdBigInt.intValue(); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == true) - { - String dropTelemetryReport = ctx.ts.dropTelemetryReport; - String telemetryReportVarOptTimeStampStr = dropTelemetryReport.substring(48,56); - BigInteger telemetryReportVarOptTimeStampBigInt = new BigInteger(telemetryReportVarOptTimeStampStr, 16); - ctx.ts.telemetryReportVarOptTimeStamp = telemetryReportVarOptTimeStampBigInt.intValue(); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == true) - { - String dropTelemetryReport = ctx.ts.dropTelemetryReport; - String telemetryReportVarOptDropCountStr = dropTelemetryReport.substring(56,64); - BigInteger telemetryReportVarOptDropCountBigInt = new BigInteger(telemetryReportVarOptDropCountStr, 16); - ctx.ts.telemetryReportVarOptDropCount = telemetryReportVarOptDropCountBigInt.intValue(); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == true) - { - String dropTelemetryReport = ctx.ts.dropTelemetryReport; - String telemetryReportVarOptHashStr = dropTelemetryReport.substring(64,96); - BigInteger telemetryReportVarOptHashBigInt = new BigInteger(telemetryReportVarOptHashStr, 16); - ctx.ts.telemetryReportVarOptHash = telemetryReportVarOptHashBigInt; - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - if(ctx.ts.isUDPv4 == true || ctx.ts.isUDPv6 == true) - { - String macAddress = ctx.ts.INTMetadataSourceMetadataOriginatingMac; - String destinationPort = ctx.ts.UDP2DstPort.toString(); - String combinedKeys = ''; - if(ctx.ts.isIPv4 == true && ctx.ts.isUDPv4 == true) - { - String IPvDestIP = ctx.ts.IPv4DestIP; - combinedKeys = combinedKeys.concat(macAddress).concat('|').concat(destinationPort).concat('|').concat(IPvDestIP).concat('|').concat('::'); - ctx.ts.dropHashInput = combinedKeys; - } - if(ctx.ts.isIPv6 == true && ctx.ts.isUDPv6 == true) - { - String IPvDestIP = ctx.ts.IPv6DestIP; - combinedKeys = combinedKeys.concat(macAddress).concat('|').concat(destinationPort).concat('|').concat(IPvDestIP).concat('|').concat('::'); - ctx.ts.dropHashInput = combinedKeys; - } - } - if(ctx.ts.isTCPv4 == true || ctx.ts.isTCPv6 == true) - { - String macAddress = ctx.ts.INTMetadataSourceMetadataOriginatingMac; - String destinationPort = ctx.ts.TCPDestPort.toString(); - String combinedKeys = ''; - if(ctx.ts.isIPv4 == true && ctx.ts.isTCPv4 == true) - { - String IPvDestIP = ctx.ts.IPv4DestIP; - combinedKeys = combinedKeys.concat(macAddress).concat('|').concat(destinationPort).concat('|').concat(IPvDestIP).concat('|').concat('::'); - ctx.ts.dropHashInput = combinedKeys; - } - if(ctx.ts.isIPv6 == true && ctx.ts.isTCPv6 == true) - { - String IPvDestIP = ctx.ts.IPv6DestIP; - combinedKeys = combinedKeys.concat(macAddress).concat('|').concat(destinationPort).concat('|').concat(IPvDestIP).concat('|').concat('::'); - ctx.ts.dropHashInput = combinedKeys; - } - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "ignore_failure" : false, - "id" : "calculateKeyHash" - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String completeSHA256Hash = ctx.ts.sha256Hash; - String subsetSHA256HashStr= completeSHA256Hash.substring(0,16); - BigInteger hashBigInt = new BigInteger(subsetSHA256HashStr, 16); - ctx.ts.dropReportHash = hashBigInt; - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if(ctx.ts.isUDPv4 == true || ctx.ts.isUDPv6 == true) - { - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDP2Checksum = tsRawData.substring(208, 212); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDP2Checksum = tsRawData.substring(248, 252); - } - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if (ctx.ts.isUDPv4 == true || ctx.ts.isUDPv6 == true) - { - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.dataII = tsRawData.substring(212, 252); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.dataII = tsRawData.substring(252, 292); - } - } - if(ctx.ts.isTCPv4 == true || ctx.ts.isTCPv6 == true) - { - String tcpNextProtoStr = '6'; - int tcpNextProtoInt = Integer.parseInt(tcpNextProtoStr); - if( etherType == '0800' && ipVersion == '4' ) - { - ctx.ts.dataII = tsRawData.substring(236, 288); - } - if(etherType == '86dd' && ipVersion == '6' ) - { - ctx.ts.dataII = tsRawData.substring(252, 304); - } - } - } - """, - "ignore_failure" : true - } - } - ] - } diff --git a/playbooks/hcp/templates/udp_and_tcp_monitor_def.json b/playbooks/hcp/templates/udp_and_tcp_monitor_def.json deleted file mode 100644 index 73774473..00000000 --- a/playbooks/hcp/templates/udp_and_tcp_monitor_def.json +++ /dev/null @@ -1,347 +0,0 @@ -{ - "type": "monitor", - "name": "DDOS-UDP-TCP-Monitor", - "enabled": true, - "schedule": { - "period": { - "interval": 1, - "unit": "MINUTES" - } - }, - "inputs": [{ - "search": { - "indices": ["packets-*"], - "query": { - "size": 0, - "query": { - "bool": { - "filter": [ - { - "range": { - "timestamp": { - "from": "{{period_end}}||-1m", - "to": "{{period_end}}", - "include_lower": true, - "include_upper": true, - "format": "epoch_millis", - "boost": 1 - } - } - } - ], - "adjust_pure_negative": true, - "boost": 1 - } - }, - "_source": { - "includes": [ - "ts.UDP2DstPort", - "ts.TCPDestPort", - "ts.IPvDestAddr", - "ts.INTMetadataSourceMetadataOriginatingMac", - "ts.IPv4SrcIP", - "ts.IPv4DestIP", - "ts.IPv6SrcIP", - "ts.IPv6DestIP", - "ts.isTCPv4", - "ts.isTCPv6", - "ts.isUDPv4", - "ts.isUDPv6" - ], - "excludes": [] - }, - "aggregations": { - "IPvDestAddrValueCount": { - "value_count": { - "field": "ts.IPvDestAddr" - } - }, - "UDP2DstPortValue": { - "terms": { - "field": "ts.UDP2DstPort", - "size": 10, - "min_doc_count": 1, - "shard_min_doc_count": 0, - "show_term_doc_count_error": false, - "order": [ - { - "_count": "desc" - }, - { - "_key": "asc" - } - ] - } - }, - "TCPDstPortValue": { - "terms": { - "field": "ts.TCPDestPort", - "size": 10, - "min_doc_count": 1, - "shard_min_doc_count": 0, - "show_term_doc_count_error": false, - "order": [ - { - "_count": "desc" - }, - { - "_key": "asc" - } - ] - } - }, - "INTMetadataSourceMetadataOriginatingMacValueCount": { - "value_count": { - "field": "ts.INTMetadataSourceMetadataOriginatingMac" - } - }, - "IPv4DestIP": { - "terms": { - "field": "ts.IPv4DestIP", - "size": 10, - "min_doc_count": 1, - "shard_min_doc_count": 0, - "show_term_doc_count_error": false, - "order": [ - { - "_count": "desc" - }, - { - "_key": "asc" - } - ] - } - }, - "UDP2DstPortValueCount": { - "value_count": { - "field": "ts.UDP2DstPort" - } - }, - "TCPDstPortValueCount": { - "value_count": { - "field": "ts.TCPDestPort" - } - }, - "IPv4SrcIP": { - "terms": { - "field": "ts.IPv4SrcIP", - "size": 10, - "min_doc_count": 1, - "shard_min_doc_count": 0, - "show_term_doc_count_error": false, - "order": [ - { - "_count": "desc" - }, - { - "_key": "asc" - } - ] - } - }, - "IPv6SrcIP": { - "terms": { - "field": "ts.IPv6SrcIP", - "size": 10, - "min_doc_count": 1, - "shard_min_doc_count": 0, - "show_term_doc_count_error": false, - "order": [ - { - "_count": "desc" - }, - { - "_key": "asc" - } - ] - } - }, - "IPv6DestIP": { - "terms": { - "field": "ts.IPv6DestIP", - "size": 10, - "min_doc_count": 1, - "shard_min_doc_count": 0, - "show_term_doc_count_error": false, - "order": [ - { - "_count": "desc" - }, - { - "_key": "asc" - } - ] - } - }, - "OriginatingMacAddrUDPv4": { - "terms": { - "field": "ts.INTMetadataSourceMetadataOriginatingMac", - "size": 10, - "min_doc_count": 1, - "shard_min_doc_count": 0, - "show_term_doc_count_error": false, - "order": [ - { - "_count": "desc" - }, - { - "_key": "asc" - } - ] - } - }, - "isTCPv4Count": { - "terms": { - "field": "ts.isTCPv4", - "size": 10, - "min_doc_count": 1, - "shard_min_doc_count": 0, - "show_term_doc_count_error": false, - "order": [ - - { - "_key": "desc" - } - ] - } - }, - "isTCPv6Count": { - "terms": { - "field": "ts.isTCPv6", - "size": 10, - "min_doc_count": 1, - "shard_min_doc_count": 0, - "show_term_doc_count_error": false, - "order": [ - - { - "_key": "desc" - } - ] - } - }, - "isUDPv4Count": { - "terms": { - "field": "ts.isUDPv4", - "size": 10, - "min_doc_count": 1, - "shard_min_doc_count": 0, - "show_term_doc_count_error": false, - "order": [ - - { - "_key": "desc" - } - ] - } - }, - "isUDPv6Count": { - "terms": { - "field": "ts.isUDPv6", - "size": 10, - "min_doc_count": 1, - "shard_min_doc_count": 0, - "show_term_doc_count_error": false, - "order": [ - - { - "_key": "desc" - } - ] - } - } - } - } - } - }], - "triggers": [ - { - "name": "DDOS-Trigger-TCPv4", - "severity": "1", - "condition": { - "script": { - "source": "if(ctx.results[0].hits.total.value > 0){ctx.results[0].aggregations.TCPDstPortValueCount.value >= 100 && ctx.results[0].aggregations.isTCPv4Count.buckets.length > 0 && ctx.results[0].aggregations.isTCPv4Count.buckets.0.key =='true' && ctx.results[0].aggregations.isTCPv4Count.buckets.0.doc_count >= 100}", - "lang": "painless" - } - }, - "actions": [{ - "name": "Sdn-WebHook-Action", - "destination_id": "sdnWebhookResponseIdTCPv4", - "message_template": { - "source":"{\"event_action\":\"trigger\", \"dst_ip\":\"{{ctx.results.0.aggregations.IPv4DestIP.buckets.0.key}}\",\"dst_port\":\"{{ctx.results.0.aggregations.TCPDstPortValue.buckets.0.key}}\",\"src_ip\":\"{{ctx.results.0.aggregations.IPv4SrcIP.buckets.0.key}}\",\"src_mac\":\"{{ctx.results.0.aggregations.OriginatingMacAddrUDPv4.buckets.0.key}}\"}", - "lang": "mustache", - "options": { - "content_type": "application/json" - } - }, - "throttle_enabled": false - }] - }, - { - "name": "DDOS-Trigger-TCPv6", - "severity": "1", - "condition": { - "script": { - "source": "if(ctx.results[0].hits.total.value > 0){ctx.results[0].aggregations.TCPDstPortValueCount.value >= 100 && ctx.results[0].aggregations.isTCPv6Count.buckets.length > 0 && ctx.results[0].aggregations.isTCPv6Count.buckets.0.key =='true' && ctx.results[0].aggregations.isTCPv6Count.buckets.0.doc_count >= 100}", - "lang": "painless" - } - }, - "actions": [{ - "name": "Sdn-WebHook-Action", - "destination_id": "sdnWebhookResponseIdTCPv6", - "message_template": { - "source":"{\"event_action\":\"trigger\", \"dst_ip\":\"{{ctx.results.0.aggregations.IPv6DestIP.buckets.0.key}}\",\"dst_port\":\"{{ctx.results.0.aggregations.TCPDstPortValue.buckets.0.key}}\",\"src_ip\":\"{{ctx.results.0.aggregations.IPv6SrcIP.buckets.0.key}}\",\"src_mac\":\"{{ctx.results.0.aggregations.OriginatingMacAddrUDPv4.buckets.0.key}}\"}", - "lang": "mustache", - "options": { - "content_type": "application/json" - } - }, - "throttle_enabled": false - }] - }, - { - "name": "DDOS-Trigger-UDPv4", - "severity": "1", - "condition": { - "script": { - "source": "if(ctx.results[0].hits.total.value > 0){ctx.results[0].aggregations.UDP2DstPortValueCount.value >= 100 && ctx.results[0].aggregations.isUDPv4Count.buckets.length > 0 && ctx.results[0].aggregations.isUDPv4Count.buckets.0.key =='true' && ctx.results[0].aggregations.isUDPv4Count.buckets.0.doc_count >= 100}", - "lang": "painless" - } - }, - "actions": [{ - "name": "Sdn-WebHook-Action", - "destination_id": "sdnWebhookResponseIdUDPv4", - "message_template": { - "source":"{\"event_action\":\"trigger\", \"dst_ip\":\"{{ctx.results.0.aggregations.IPv4DestIP.buckets.0.key}}\",\"dst_port\":\"{{ctx.results.0.aggregations.UDP2DstPortValue.buckets.0.key}}\",\"src_ip\":\"{{ctx.results.0.aggregations.IPv4SrcIP.buckets.0.key}}\",\"src_mac\":\"{{ctx.results.0.aggregations.OriginatingMacAddrUDPv4.buckets.0.key}}\"}", - "lang": "mustache", - "options": { - "content_type": "application/json" - } - }, - "throttle_enabled": false - }] - }, - { - "name": "DDOS-Trigger-UDPv6", - "severity": "1", - "condition": { - "script": { - "source": "if(ctx.results[0].hits.total.value > 0){ctx.results[0].aggregations.UDP2DstPortValueCount.value >= 100 && ctx.results[0].aggregations.isUDPv6Count.buckets.length > 0 && ctx.results[0].aggregations.isUDPv6Count.buckets.0.key =='true' && ctx.results[0].aggregations.isUDPv6Count.buckets.0.doc_count >= 100}", - "lang": "painless" - } - }, - "actions": [{ - "name": "Sdn-WebHook-Action", - "destination_id": "sdnWebhookResponseIdUDPv6", - "message_template": { - "source":"{\"event_action\":\"trigger\", \"dst_ip\":\"{{ctx.results.0.aggregations.IPv6DestIP.buckets.0.key}}\",\"dst_port\":\"{{ctx.results.0.aggregations.UDP2DstPortValue.buckets.0.key}}\",\"src_ip\":\"{{ctx.results.0.aggregations.IPv6SrcIP.buckets.0.key}}\",\"src_mac\":\"{{ctx.results.0.aggregations.OriginatingMacAddrUDPv4.buckets.0.key}}\"}", - "lang": "mustache", - "options": { - "content_type": "application/json" - } - }, - "throttle_enabled": false - }] - } - ] -} diff --git a/playbooks/hcp/toggle_ae_monitors.yml b/playbooks/hcp/toggle_ae_monitors.yml deleted file mode 100644 index 90d9f03d..00000000 --- a/playbooks/hcp/toggle_ae_monitors.yml +++ /dev/null @@ -1,49 +0,0 @@ - -- hosts: ae - gather_facts: no - tasks: - - name: Get monitor ID for DDOS-UDP-TCP monitor to update via monitor name - uri: - url: "http://localhost:9200/_opendistro/_alerting/monitors/_search" - method: GET - return_content: yes - body: "{{ lookup('file','./templates/query_monitor_id.json') }}" - body_format: json - register: get_monitor_id_response - - - name: Show retrieved Monitor definition - debug: - var: get_monitor_id_response.json - - - name: Set fact for source from retrieved Monitor definition - set_fact: - monitor_put_original_body: "{{ get_monitor_id_response.json.hits.hits[0]._source }}" - - - name: Change enabled to passed as argument value. - set_fact: - monitor_updated_def: "{{ monitor_put_original_body|combine({'enabled': false }, recursive=True) }}" - when: not toggle | bool - - - name: Change enabled to passed as argument value. - set_fact: - monitor_updated_def: "{{ monitor_put_original_body|combine({'enabled': true }, recursive=True) }}" - when: toggle | bool - - - name: Debug for monitor_updated_def - debug: - var: monitor_updated_def - - - name: PUT call to update Monitor definition in Elasticsearch - uri: - url: "http://localhost:9200/_opendistro/_alerting/monitors/{{get_monitor_id_response.json.hits.hits[0]._id}}" - method: PUT - return_content: yes - status_code: 200 - body: "{{ monitor_updated_def }}" - body_format: json - register: updatedMonitorDefResponse - - - name: Fetch updated monitor definition. - debug: - msg: " Updated Monitor Definition : {{ updatedMonitorDefResponse }}" - diff --git a/playbooks/mininet/ae_tunnel.yml b/playbooks/mininet/ae_tunnel.yml deleted file mode 100644 index e5f9238b..00000000 --- a/playbooks/mininet/ae_tunnel.yml +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright (c) 2020 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -- hosts: all - gather_facts: no - become: yes - tasks: - - name: Create GRE tunnel - command: > - ip link add {{ tunnel_name }} type gretap - local {{ local_ip }} - remote {{ remote_ip }} - - - name: Activate tunnel - command: > - ip link set {{ tunnel_name }} up diff --git a/playbooks/mininet/compile_p4.yml b/playbooks/mininet/compile_p4.yml deleted file mode 100644 index bf64a6d7..00000000 --- a/playbooks/mininet/compile_p4.yml +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -- hosts: "{{ host_val | default('all') }}" - gather_facts: no - become: yes - - tasks: - - name: Ensure {{ remote_scripts_dir }} directory has been created - file: - path: "{{ remote_scripts_dir }}/p4" - state: directory - - - name: Compile P4 programs {{ p4_progs }} - command: > - p4c-bm2-ss - --p4v 16 - --p4runtime-format text - --p4runtime-file {{ remote_scripts_dir }}/p4/{{ item }}.p4info - -o {{ remote_scripts_dir }}/p4/{{ item }}.json -D BMV2=foo - {{ trans_sec_dir }}/p4/{{ item }}/{{ item }}.p4 - register: cmd_out - changed_when: cmd_out is not failed - loop: "{{ p4_progs }}" diff --git a/playbooks/mininet/generate_inventory.yml b/playbooks/mininet/generate_inventory.yml deleted file mode 100644 index 91cd4bea..00000000 --- a/playbooks/mininet/generate_inventory.yml +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -- hosts: all - gather_facts: no - tasks: - - name: Create ansible inventory file {{ remote_inventory_file }} - file: - path: "{{ remote_inventory_file }}" - state: touch - - - name: Create inventory variables heading at {{ remote_inventory_file }} - lineinfile: - path: "{{ remote_inventory_file }}" - line: "[all:vars]" - state: present - - - name: Add variable entries to ansible inventory at {{ remote_inventory_file }} - lineinfile: - path: "{{ remote_inventory_file }}" - line: "{{ item }}" - state: present - loop: - - "trans_sec_dir={{ src_dir }}" - - "srvc_log_dir={{ remote_srvc_log_dir }}" - - "log_level={{ service_log_level }}" - - "log_dir={{ remote_srvc_log_dir }}" - - "remote_scripts_dir={{ remote_scripts_dir }}" - - "remote_ansible_inventory={{ remote_inventory_file }}" - - "topo_file_loc={{ topo_file }}" - - "mn_host_ip={{ mn_host }}" - - "sdn_ip={{ sdn_host }}" - - "sdn_port={{ sdn_port }}" - - "ae_ip={{ ae_host }}" - - "sdn_dev_intf={{ sdn_dev_intf }}" - - "ae_dev_intf={{ ae_dev_intf }}" - - "ae_monitor_intf={{ ae_monitor_intf }}" - - "sdn_url=http://{{ sdn_host }}:{{ sdn_port }}" - - "clone_egress_port={{ clone_egress_port }}" - - "scenario_name={{ scenario_name }}" diff --git a/playbooks/mininet/generate_topology.yml b/playbooks/mininet/generate_topology.yml deleted file mode 100644 index a248fad0..00000000 --- a/playbooks/mininet/generate_topology.yml +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -- hosts: "{{ host_val | default('all') }}" - become: yes - - gather_facts: yes - - vars: - switch_mac: "{{ ansible_facts['eth0']['macaddress'] }}" - - tasks: - - name: Ensure {{ remote_scripts_dir }} scripts directory has been created - file: - path: "{{ remote_scripts_dir }}" - state: directory - - - name: Apply topology configuration template {{ topology_template }} to {{ topo_file_loc }} - become: yes - template: - src: "{{ topology_template }}" - dest: "{{ topo_file_loc }}" diff --git a/playbooks/mininet/local_inventory.yml b/playbooks/mininet/local_inventory.yml deleted file mode 100644 index 65d7caa3..00000000 --- a/playbooks/mininet/local_inventory.yml +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -- hosts: localhost - gather_facts: no - tasks: - - name: copy variable configuration file - vars: - public_ip: "{{ public_ip }}" - template: - src: templates/local_inventory.ini.j2 - dest: "{{ local_inventory }}" - delegate_to: localhost diff --git a/playbooks/mininet/set_ip-full.yml b/playbooks/mininet/set_ip-full.yml deleted file mode 100644 index 33ce05dd..00000000 --- a/playbooks/mininet/set_ip-full.yml +++ /dev/null @@ -1,93 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -- name: Set node facts - set_fact: - north_node: "{{ item.north_node | default(None) }}" - south_node: "{{ item.south_node | default(None) }}" - south_facing_port: "{{ item.south_facing_port | default(None) }}" - north_facing_port: "{{ item.north_facing_port | default(None) }}" - host: "{{ topo_dict.hosts.get(item.south_node) }}" - south_facing_ip: "{{ item.south_facing_ip | default(None) }}" - south_switch: "{{ topo_dict.switches.get(item.south_node) }}" - north_switch: "{{ topo_dict.switches.get(item.north_node) }}" - intf_name: None - -- name: Find north node if south didn't exist - set_fact: - host: "{{ topo_dict.hosts.get(item.north_node) }}" - when: not host - -- name: Determine interface name for south_facing_port - set_fact: - intf_name: "{{ north_node | default('') }}-eth{{ south_facing_port | default('') }}" - when: south_facing_port and not north_facing_port - -- name: Determine interface name for south_facing_port - set_fact: - intf_name: "{{ south_node | default('') }}-eth{{ north_facing_port | default('') }}" - when: north_facing_port and not south_facing_port - -- name: Determine interface name for gateway north-facing port - set_fact: - intf_name: "{{ south_node | default('') }}-eth{{ north_facing_port | default('') }}" - when: north_facing_port and south_facing_port and south_switch - -- name: Set public IP on the gateway - block: - - name: Set public IP for intf {{ intf_name }} - become: yes - command: "ip addr add {{ south_switch.public_ip }}/24 dev {{ intf_name }}" - when: south_switch.public_ip is defined and north_switch - -- name: Setup local IPs on the gateway - block: - - name: Set local IP for intf {{ intf_name }} - become: yes - command: "ip addr add {{ host.switch_ip }}/30 dev {{ intf_name }}" - when: host and north_switch.public_ip is defined - -- name: Setup local IP on the core - block: - - name: Set local IP for core intf {{ intf_name }} - become: yes - command: "ip addr add {{ south_switch.public_ip }}/24 dev {{ intf_name }}" - when: host and south_switch.public_ip is defined - -- name: Install ifmetric - become: yes - apt: - update_cache: yes - name: - - ifmetric - register: apt_rc - retries: 3 - delay: 10 - until: apt_rc is not failed - -- name: Set route priority - become: yes - command: "ifmetric {{ intf_name }} 50" - register: cmd_out - changed_when: cmd_out is not failed - retries: 3 - delay: 5 - -- name: Set route priority for core-inet link - become: yes - command: "ifmetric {{ intf_name }} 25" - register: cmd_out - changed_when: cmd_out is not failed - retries: 3 - delay: 5 - when: host and south_switch.public_ip is defined diff --git a/playbooks/mininet/set_ip-lab_trial.yml b/playbooks/mininet/set_ip-lab_trial.yml deleted file mode 100644 index 67cd48b9..00000000 --- a/playbooks/mininet/set_ip-lab_trial.yml +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -- name: Set node facts - set_fact: - north_node: "{{ item.north_node | default(None) }}" - south_node: "{{ item.south_node | default(None) }}" - south_facing_port: "{{ item.south_facing_port | default(None) }}" - north_facing_port: "{{ item.north_facing_port | default(None) }}" - host: "{{ topo_dict.hosts.get(item.south_node) }}" - south_switch: "{{ topo_dict.switches.get(item.south_node) }}" - north_switch: "{{ topo_dict.switches.get(item.north_node) }}" - north_facing_ip: "{{ item.north_facing_ip | default(None) }}" - south_facing_ip: "{{ item.south_facing_ip | default(None) }}" - intf_name: None - -- name: Find north node if south didn't exist - set_fact: - host: "{{ topo_dict.hosts.get(item.north_node) }}" - when: not host - -- name: Determine interface name for south_facing_port - set_fact: - intf_name: "{{ north_node | default('') }}-eth{{ south_facing_port | default('') }}" - when: south_facing_port and not north_facing_port - -- name: Determine interface name for north_facing_port - set_fact: - intf_name: "{{ south_node | default('') }}-eth{{ north_facing_port | default('') }}" - when: north_facing_port and not south_facing_port - -- name: Set north-facing IP on the core - {{ intf_name }} - become: yes - command: "ip addr add {{ north_facing_ip }}/30 dev {{ intf_name }}" - when: host and south_switch and north_facing_ip is defined - -- name: Setup south-facing IP on the aggregate - {{ intf_name }} - become: yes - command: "ip addr add {{ south_facing_ip }}/30 dev {{ intf_name }}" - when: host and north_switch and south_facing_ip is defined diff --git a/playbooks/mininet/set_ip-single-switch.yml b/playbooks/mininet/set_ip-single-switch.yml deleted file mode 100644 index 0fa8244e..00000000 --- a/playbooks/mininet/set_ip-single-switch.yml +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -- name: Set node facts - set_fact: - north_node: "{{ item.north_node | default(None) }}" - south_node: "{{ item.south_node | default(None) }}" - south_facing_port: "{{ item.south_facing_port | default(None) }}" - north_facing_port: "{{ item.north_facing_port | default(None) }}" - host: "{{ topo_dict.hosts.get(item.south_node) }}" - intf_name: None - -- name: Find north node if south didn't exist - set_fact: - host: "{{ topo_dict.hosts.get(item.north_node) }}" - when: not host - -- name: Determine interface name for south_facing_port - set_fact: - intf_name: "{{ north_node | default('') }}-eth{{ south_facing_port | default('') }}" - when: south_facing_port and not north_facing_port - -- name: Determine interface name for south_facing_port - set_fact: - intf_name: "{{ south_node | default('') }}-eth{{ north_facing_port | default('') }}" - when: north_facing_port and not south_facing_port - -- name: Set IP for intf {{ intf_name }} - become: yes - command: "ip addr add {{ host.switch_ip }}/30 dev {{ intf_name }}" - when: host diff --git a/playbooks/mininet/setup-full.yml b/playbooks/mininet/setup-full.yml deleted file mode 100644 index c93c207d..00000000 --- a/playbooks/mininet/setup-full.yml +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -- import_playbook: start_mininet.yml - vars: - host_val: localhost - service_name: tps-mininet - srvc_desc: 'Mininet for TPS' - local_srvc_script_tmplt_file: ../general/templates/mininet_service.sh.j2 - srvc_start_pause_time: 20 - port_to_wait: 50051 - p4_progs: - - core - - aggregate - - gateway - set_ip_file: set_ip-full.yml - topology_template: templates/topology_template-full.yml.j2 - -- import_playbook: ../general/start_service.yml - vars: - host_val: localhost - service_name: tps-sdn - srvc_desc: 'SDN' - local_srvc_script_tmplt_file: ../general/templates/sdn_controller.sh.j2 - srvc_start_pause_time: 15 - port_to_wait: "{{ sdn_port }}" - p4_platform: bmv2 - load_p4: True - -- import_playbook: ../general/start_service.yml - vars: - host_val: localhost - service_name: tps-ae - srvc_desc: 'TPS-AE' - srvc_type: 'SIMPLE' - local_srvc_script_tmplt_file: ../general/templates/ae_service.sh.j2 - monitor_intf: "{{ ae_monitor_intf }}" diff --git a/playbooks/mininet/setup-lab_trial.yml b/playbooks/mininet/setup-lab_trial.yml deleted file mode 100644 index 5986347a..00000000 --- a/playbooks/mininet/setup-lab_trial.yml +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -- import_playbook: start_mininet.yml - vars: - host_val: localhost - service_name: tps-mininet - srvc_desc: 'Mininet for {{ scenario_name }}.p4' - local_srvc_script_tmplt_file: ../general/templates/mininet_service.sh.j2 - srvc_start_pause_time: 20 - port_to_wait: 50051 - p4_progs: - - core - - aggregate - set_ip_file: set_ip-lab_trial.yml - topology_template: templates/topology_template-lab_trial.yml.j2 - load_p4: False - -- import_playbook: ../general/start_service.yml - vars: - host_val: localhost - service_name: tps-sdn - srvc_desc: 'SDN' - local_srvc_script_tmplt_file: ../general/templates/sdn_controller.sh.j2 - srvc_start_pause_time: 15 - port_to_wait: "{{ sdn_port }}" - p4_platform: bmv2 - load_p4: True diff --git a/playbooks/mininet/setup-single_switch.yml b/playbooks/mininet/setup-single_switch.yml deleted file mode 100644 index 4435df3b..00000000 --- a/playbooks/mininet/setup-single_switch.yml +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -- import_playbook: start_mininet.yml - vars: - host_val: localhost - service_name: tps-mininet - srvc_desc: 'Mininet for {{ scenario_name }}.p4' - local_srvc_script_tmplt_file: ../general/templates/mininet_service.sh.j2 - srvc_start_pause_time: 20 - port_to_wait: 50051 - p4_progs: - - "{{ scenario_name }}" - set_ip_file: set_ip-single-switch.yml - topology_template: "templates/topology_template-single_switch.yml.j2" - -- import_playbook: ../general/start_service.yml - vars: - host_val: localhost - service_name: tps-sdn - srvc_desc: 'SDN' - local_srvc_script_tmplt_file: ../general/templates/sdn_controller.sh.j2 - srvc_start_pause_time: 15 - port_to_wait: "{{ sdn_port }}" - p4_platform: bmv2 - load_p4: True diff --git a/playbooks/mininet/setup_ae2mini_tunnel.yml b/playbooks/mininet/setup_ae2mini_tunnel.yml deleted file mode 100644 index 0c5dcece..00000000 --- a/playbooks/mininet/setup_ae2mini_tunnel.yml +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright (c) 2020 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -- import_playbook: ae_tunnel.yml diff --git a/playbooks/mininet/setup_host.yml b/playbooks/mininet/setup_host.yml deleted file mode 100644 index abd383af..00000000 --- a/playbooks/mininet/setup_host.yml +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -- import_playbook: ../general/setup_source.yml - vars: - python_unit_tests: true - -- import_playbook: generate_inventory.yml diff --git a/playbooks/mininet/setup_mini2ae_tunnel.yml b/playbooks/mininet/setup_mini2ae_tunnel.yml deleted file mode 100644 index 92286085..00000000 --- a/playbooks/mininet/setup_mini2ae_tunnel.yml +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright (c) 2020 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -- import_playbook: ae_tunnel.yml - -- hosts: all - gather_facts: no - become: yes - tasks: - - name: Install daemonlogger - apt: - name: daemonlogger - -- import_playbook: ../general/start_service.yml - vars: - host_val: all - service_name: tps-daemon-logger - srvc_desc: 'TPS-CLONE-INTF-MIRROR' - local_srvc_script_tmplt_file: ../general/templates/daemonlogger_service.sh.j2 diff --git a/playbooks/mininet/start_mininet.yml b/playbooks/mininet/start_mininet.yml deleted file mode 100644 index 8d0918cc..00000000 --- a/playbooks/mininet/start_mininet.yml +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -- import_playbook: generate_topology.yml - vars: - host_val: localhost - topology_template: "{{ topology_template }}" - -- import_playbook: compile_p4.yml - vars: - host_val: localhost - p4_progs: "{{ p4_progs }}" - -- import_playbook: ../general/start_service.yml - vars: - host_val: localhost - -- hosts: localhost - gather_facts: no - become: yes - - vars: - topo_file_dest: "{{ topo_file_remote_loc | default('/tmp/mininet-topology.yaml') }}" - - tasks: - - name: Fetch topology {{ topo_file_loc }} - fetch: - src: "{{ topo_file_loc }}" - dest: "{{ topo_file_dest }}" - flat: yes - - - name: Read topology - set_fact: - topo_dict: "{{ lookup('file', topo_file_dest) | from_yaml }}" - - - include_tasks: "{{ set_ip_file }}" - loop: "{{ topo_dict.links }}" - - - name: Set /etc/hosts - become: yes - lineinfile: - path: /etc/hosts - line: "{{ item.ip }} {{ item.name }}" - state: present - loop: "{{ topo_dict.hosts.values() | list }}" - - - name: Add entries to ansible inventory at {{ remote_ansible_inventory }} - lineinfile: - path: "{{ remote_ansible_inventory }}" - insertbefore: BOF - line: "{{ item }}" - state: present - loop: "{{ topo_dict.hosts.keys() | list }}" - - - name: Add localhost to {{ remote_ansible_inventory }} - lineinfile: - path: "{{ remote_ansible_inventory }}" - insertbefore: BOF - line: "localhost" - state: present diff --git a/playbooks/mininet/templates/local_inventory.ini.j2 b/playbooks/mininet/templates/local_inventory.ini.j2 deleted file mode 100644 index 74ee52d5..00000000 --- a/playbooks/mininet/templates/local_inventory.ini.j2 +++ /dev/null @@ -1,20 +0,0 @@ -{{ public_ip }} -[all:vars] -remote_inventory_file={{ remote_inventory_file | default('/home/ubuntu/transparent-security.ini')}} -src_dir={{ src_dir | default('/home/ubuntu/transparent-security') }} -remote_srvc_log_dir={{ remote_srvc_log_dir | default('/var/log/transparent-security') }} -service_log_level={{ service_log_level | default('INFO') }} -log_dir={{ remote_srvc_log_dir | default('/var/log/transparent-security') }} -trans_sec_dir={{ trans_sec_dir | default('/home/ubuntu/transparent-security') }} -remote_scripts_dir={{ remote_scripts_dir | default('/etc/transparent-security') }} -topo_file={{ topo_file_loc | default('/etc/transparent-security/mininet-sim-topology.yaml') }} -forwarding_daemon_file={{ forwarding_daemon_file | default('/etc/transparent-security/forwarding-daemons.yml') }} -mn_host={{ mn_host | default('localhost') }} -sdn_host={{ sdn_host | default('localhost') }} -sdn_port={{ sdn_port | default('9998') }} -ae_host={{ ae_host | default('localhost') }} -sdn_dev_intf={{ sdn_dev_intf | default('lo') }} -ae_dev_intf={{ ae_dev_intf | default('lo') }} -ae_monitor_intf={{ ae_monitor_intf | default('core-eth3') }} -clone_egress_port={{ clone_egress_port | default('3') }} -scenario_name={{ scenario_name | default('full') }} diff --git a/playbooks/mininet/templates/topology_template-full.yml.j2 b/playbooks/mininet/templates/topology_template-full.yml.j2 deleted file mode 100755 index 7643cc0f..00000000 --- a/playbooks/mininet/templates/topology_template-full.yml.j2 +++ /dev/null @@ -1,284 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -# IP values must be the third address in an IPv4 /30 CIDR block for running CI -hosts: - Camera1: - name: Camera1 - id: 5 - switch_ip: 192.168.1.1 - switch_mac: 00:00:01:01:01:00 - ip: 192.168.1.2 - ipv6: 0000:0000:0000:0000:0000:0001:0001:0002 - ip_port: any - mac: 00:00:00:00:01:01 - type: webcam - NAS1: - name: NAS1 - id: 6 - switch_ip: 192.168.1.5 - switch_mac: 00:00:01:01:02:00 - ip: 192.168.1.6 - ipv6: 0000:0000:0000:0000:0000:0001:0001:0003 - ip_port: any - mac: 00:00:00:00:01:02 - type: nas - Game1: - name: Game1 - id: 7 - switch_ip: 192.168.1.9 - switch_mac: 00:00:01:01:03:00 - ip: 192.168.1.10 - ipv6: 0000:0000:0000:0000:0000:0001:0001:0004 - ip_port: any - mac: 00:00:00:00:01:03 - type: console - Camera2: - name: Camera2 - id: 8 - switch_ip: 192.168.2.1 - switch_mac: 00:00:02:02:01:00 - ipv6: 0000:0000:0000:0000:0000:0001:0002:0002 - ip: 192.168.2.2 - ip_port: any - mac: 00:00:00:00:02:01 - type: webcam - Game2: - name: Game2 - id: 9 - switch_ip: 192.168.2.5 - switch_mac: 00:00:02:02:02:00 - ip: 192.168.2.6 - ipv6: 0000:0000:0000:0000:0000:0001:0002:0003 - ip_port: any - mac: 00:00:00:00:02:02 - type: console - Camera3: - name: Camera3 - id: 10 - switch_ip: 192.168.3.1 - switch_mac: 00:00:03:03:01:00 - ip: 192.168.3.2 - ipv6: 0000:0000:0000:0000:0000:0001:0003:0002 - ip_port: any - mac: 00:00:00:00:03:01 - type: webcam - Game3: - name: Game3 - id: 11 - switch_ip: 192.168.3.5 - switch_mac: 00:00:03:03:02:00 - ip: 192.168.3.6 - ipv6: 0000:0000:0000:0000:0000:0001:0003:0003 - ip_port: any - mac: 00:00:00:00:03:02 - type: console - inet: - name: inet - id: 12 - switch_ip: 172.32.1.4 - ip: 172.32.1.5 - ipv6: 0000:0000:0000:0000:0000:0002:0001:0002 - ip_port: any - mac: 00:00:00:00:05:07 - type: target-server -switches: -# TODO - Determine why changes to the ID in any of the switch id values causes a GRPC exception in the SDN controller upon startup - gateway1: - name: gateway1 - id: 2 - runtime_p4info: {{ remote_scripts_dir }}/p4/gateway.p4info - runtime_json: {{ remote_scripts_dir }}/p4/gateway.json - public_ip: 172.32.1.1 - type: gateway - mac: 00:00:00:02:01:00 - grpc: {{ mn_host_ip }}:50053 - ipv6_term_host: inet - multicast_entries: - - egress_port: 1 - instance: 1 - - egress_port: 2 - instance: 1 - - egress_port: 3 - instance: 1 - - egress_port: 4 - instance: 1 - gateway2: - name: gateway2 - id: 3 - runtime_p4info: {{ remote_scripts_dir }}/p4/gateway.p4info - runtime_json: {{ remote_scripts_dir }}/p4/gateway.json - public_ip: 172.32.1.2 - type: gateway - mac: 00:00:00:02:02:00 - grpc: {{ mn_host_ip }}:50054 - ipv6_term_host: inet - multicast_entries: - - egress_port: 1 - instance: 1 - - egress_port: 2 - instance: 1 - - egress_port: 3 - instance: 1 - gateway3: - name: gateway3 - id: 4 - runtime_p4info: {{ remote_scripts_dir }}/p4/gateway.p4info - runtime_json: {{ remote_scripts_dir }}/p4/gateway.json - public_ip: 172.32.1.3 - type: gateway - mac: 00:00:00:02:03:00 - grpc: {{ mn_host_ip }}:50055 - ipv6_term_host: inet - multicast_entries: - - egress_port: 1 - instance: 1 - - egress_port: 2 - instance: 1 - - egress_port: 3 - instance: 1 - aggregate: - name: aggregate - id: 0 - runtime_p4info: {{ remote_scripts_dir }}/p4/aggregate.p4info - runtime_json: {{ remote_scripts_dir }}/p4/aggregate.json - type: aggregate - mac: 00:00:00:02:04:00 - grpc: {{ mn_host_ip }}:50051 - ipv6_term_host: inet - multicast_entries: - - egress_port: 1 - instance: 1 - - egress_port: 2 - instance: 1 - - egress_port: 3 - instance: 1 - - egress_port: 4 - instance: 1 - core: - name: core - id: 1 - runtime_p4info: {{ remote_scripts_dir }}/p4/core.p4info - runtime_json: {{ remote_scripts_dir }}/p4/core.json - clone_egress: {{ clone_egress_port }} - public_ip: 172.32.1.4 - telemetry_rpt: - type: external - name: ae - type: core - mac: 00:00:00:03:05:00 - grpc: {{ mn_host_ip }}:50052 - ipv6_term_host: inet - multicast_entries: - - egress_port: 1 - instance: 1 - - egress_port: 2 - instance: 1 -external: - ae: - name: ae - id: {{ ae_dev_intf }} - ip: {{ ae_ip }} - ip_port: 0 - mac: {{ switch_mac }} - sdn: - name: sdn - id: {{ sdn_dev_intf }} - ip: {{ sdn_ip }} - mac: {{ switch_mac }} -links: - - north_node: gateway1 - south_node: Camera1 - south_facing_port: 1 - south_facing_mac: 00:00:01:01:01:00 - south_facing_ip: 192.168.1.1 - latency: 0ms - bandwidth: null - - north_node: gateway1 - south_node: NAS1 - south_facing_port: 2 - south_facing_mac: 00:00:01:01:02:00 - south_facing_ip: 192.168.1.5 - latency: 0ms - bandwidth: null - - north_node: gateway1 - south_node: Game1 - south_facing_port: 3 - south_facing_mac: 00:00:01:01:03:00 - south_facing_ip: 192.168.1.9 - latency: 0ms - bandwidth: null - - north_node: gateway2 - south_node: Camera2 - south_facing_port: 1 - south_facing_mac: 00:00:02:02:01:00 - south_facing_ip: 192.168.2.1 - latency: 0ms - bandwidth: null - - north_node: gateway2 - south_node: Game2 - south_facing_port: 2 - south_facing_mac: 00:00:02:02:02:00 - south_facing_ip: 192.168.2.5 - latency: 0ms - bandwidth: null - - north_node: gateway3 - south_node: Camera3 - south_facing_port: 1 - south_facing_mac: 00:00:03:03:01:00 - south_facing_ip: 192.168.3.1 - latency: 0ms - bandwidth: null - - north_node: gateway3 - south_node: Game3 - south_facing_port: 2 - south_facing_mac: 00:00:03:03:02:00 - south_facing_ip: 192.168.3.5 - latency: 0ms - bandwidth: null - - north_node: aggregate - south_node: gateway1 - south_facing_port: 1 - north_facing_port: 4 - latency: 0ms - bandwidth: null - - north_node: aggregate - south_node: gateway2 - south_facing_port: 2 - north_facing_port: 3 - latency: 0ms - bandwidth: null - - north_node: aggregate - south_node: gateway3 - south_facing_port: 3 - north_facing_port: 3 - latency: 0ms - bandwidth: null - - north_node: core - south_node: aggregate - south_facing_port: 1 - north_facing_port: 4 - latency: 0ms - bandwidth: null - - north_node: inet - south_node: core - north_facing_port: 2 - latency: 0ms - bandwidth: null - - north_node: ae - south_node: core - north_facing_port: 3 - latency: 0ms - bandwidth: null - l2ptr: 53261 diff --git a/playbooks/mininet/templates/topology_template-lab_trial.yml.j2 b/playbooks/mininet/templates/topology_template-lab_trial.yml.j2 deleted file mode 100644 index 9b8837b8..00000000 --- a/playbooks/mininet/templates/topology_template-lab_trial.yml.j2 +++ /dev/null @@ -1,121 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -# IP values must be the third address in an IPv4 /30 CIDR block for running CI -hosts: - host1: - name: host1 - id: 5 - switch_ip: 192.168.1.1 - ip: 192.168.1.2 - ipv6: 0000:0000:0000:0000:0000:0001:0001:0002 - ip_port: any - mac: 00:00:00:00:01:01 - type: webcam - host2: - name: host2 - id: 6 - switch_ip: 192.168.1.5 - ip: 192.168.1.6 - ipv6: 0000:0000:0000:0000:0000:0001:0001:0003 - ip_port: any - mac: 00:00:00:00:01:02 - type: nas - inet: - name: inet - id: 12 - switch_ip: 192.168.1.9 - ip: 192.168.1.10 - ipv6: 0000:0000:0000:0000:0000:0002:0001:0002 - ip_port: any - mac: 00:00:00:00:05:07 - type: target-server - ae: - name: ae - id: 13 - switch_ip: 10.0.1.8 - ip: 10.0.1.9 - ipv6: 0000:0000:0000:0000:0000:0002:0001:0003 - ip_port: any - mac: 00:00:00:00:05:08 - type: server -switches: - aggregate: - name: aggregate - id: 0 - runtime_p4info: {{ remote_scripts_dir }}/p4/aggregate.p4info - runtime_json: {{ remote_scripts_dir }}/p4/aggregate.json - type: aggregate - mac: 00:00:00:02:04:00 - grpc: 127.0.0.1:50051 - ipv6_term_host: inet - multicast_entries: - - egress_port: 1 - instance: 1 - - egress_port: 2 - instance: 1 - - egress_port: 3 - instance: 1 - core: - name: core - id: 1 - runtime_p4info: {{ remote_scripts_dir }}/p4/core.p4info - runtime_json: {{ remote_scripts_dir }}/p4/core.json - clone_egress: 3 - telemetry_rpt: - type: host - name: ae - type: core - mac: 00:00:00:03:05:00 - grpc: 127.0.0.1:50052 - ipv6_term_host: inet - multicast_entries: - - egress_port: 1 - instance: 1 - - egress_port: 2 - instance: 1 - - egress_port: 3 - instance: 1 -links: - - north_node: aggregate - south_node: host1 - south_facing_port: 1 - south_facing_ip: 192.168.1.1 - latency: 0ms - bandwidth: null - - north_node: aggregate - south_node: host2 - south_facing_port: 2 - south_facing_ip: 192.168.1.5 - latency: 0ms - bandwidth: null - - north_node: core - south_node: aggregate - south_facing_port: 1 - north_facing_port: 3 - latency: 0ms - bandwidth: null - - north_node: inet - south_node: core - north_facing_port: 2 - north_facing_ip: 192.168.1.9 - latency: 0ms - bandwidth: null - - north_node: ae - south_node: core - north_facing_port: 3 - north_facing_ip: 10.0.1.8 - latency: 0ms - bandwidth: null - l2ptr: 53261 diff --git a/playbooks/mininet/templates/topology_template-single_switch.yml.j2 b/playbooks/mininet/templates/topology_template-single_switch.yml.j2 deleted file mode 100644 index 556ddc95..00000000 --- a/playbooks/mininet/templates/topology_template-single_switch.yml.j2 +++ /dev/null @@ -1,88 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -# IP values must be the third address in an IPv4 /30 CIDR block for running CI -hosts: - host1: - name: host1 - id: 1 - switch_ip: 192.168.1.1 - ip: 192.168.1.2 - ipv6: 0000:0000:0000:0000:0000:0001:0001:0002 - ip_port: any - mac: 00:00:00:00:01:01 - type: webcam - host2: - name: host2 - id: 2 - switch_ip: 172.32.1.4 - ip: 172.32.1.5 - ipv6: 0000:0000:0000:0000:0000:0002:0001:0002 - ip_port: any - mac: 00:00:00:00:05:07 - type: target-server -{% if p4_progs[0] != 'aggregate' %} - clone: - name: clone - id: 3 - switch_ip: 10.0.1.8 - ip: 10.0.1.9 - ipv6: 0000:0000:0000:0000:0000:0002:0001:0003 - ip_port: any - mac: 00:00:00:00:05:08 - type: server -{% endif %} -switches: - {{ p4_progs[0] }}: - name: {{ p4_progs[0] }} - id: 0 - runtime_p4info: {{ remote_scripts_dir }}/p4/{{ p4_progs[0] }}.p4info - runtime_json: {{ remote_scripts_dir }}/p4/{{ p4_progs[0] }}.json - clone_egress: {{ clone_egress_port }} - public_ip: 172.32.1.4 - subnet: 10.0.1.0 - type: {{ p4_progs[0] }} - mac: 00:00:00:01:01:00 - grpc: {{ mn_host_ip }}:50051 - telemetry_rpt: - type: host - name: clone - ipv6_term_host: host2 -{% if p4_progs[0] != 'gateway' %} - multicast_entries: - - egress_port: 1 - instance: 1 - - egress_port: 2 - instance: 1 - - egress_port: 3 - instance: 1 -{% endif %} -links: - - north_node: {{ p4_progs[0] }} - south_node: host1 - south_facing_port: 1 - latency: 0ms - bandwidth: null - - north_node: host2 - south_node: {{ p4_progs[0] }} - north_facing_port: 2 - latency: 0ms - bandwidth: null -{% if p4_progs[0] != 'aggregate' %} - - north_node: clone - south_node: {{ p4_progs[0] }} - north_facing_port: 3 - latency: 0ms - bandwidth: null -{% endif %} diff --git a/playbooks/scenarios/full/all-tofino.yml b/playbooks/scenarios/full/all-tofino.yml deleted file mode 100644 index 24e5613b..00000000 --- a/playbooks/scenarios/full/all-tofino.yml +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Simple scenario where packets are sent through 3 devices and only the last -# one will be demonstrating dropped packets ---- -# Discover routes for full scenario -- import_playbook: discover_full.yml - vars: - host: gateway1 - interfaces: - - veth3 - -- import_playbook: discover_full.yml - vars: - host: gateway2 - interfaces: - - veth2 - -- import_playbook: discover_full.yml - vars: - host: gateway3 - interfaces: - - veth2 - -# UDP Flood Scenario IPv4 -- import_playbook: packet-flood.yml - vars: - scenario_send_protocol: UDP - scenario_send_ip_version: 4 - - -# UDP Flood Scenario IPv6 -- import_playbook: packet-flood.yml - vars: - scenario_send_protocol: UDP - scenario_send_ip_version: 6 - -# SYN Flood Scenario IPv4 -- import_playbook: packet-flood.yml - vars: - scenario_send_protocol: TCP - scenario_send_ip_version: 4 - -# SYN Flood Scenario IPv6 -- import_playbook: packet-flood.yml - vars: - scenario_send_protocol: TCP - scenario_send_ip_version: 6 diff --git a/playbooks/scenarios/full/all.yml b/playbooks/scenarios/full/all.yml deleted file mode 100644 index 4ea602f1..00000000 --- a/playbooks/scenarios/full/all.yml +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Simple scenario where packets are sent through 3 devices and only the last -# one will be demonstrating dropped packets ---- -# Discover routes for full scenario -- import_playbook: discover_full.yml - vars: - host: localhost - interfaces: - - gateway1-eth4 - - gateway2-eth3 - - gateway3-eth3 - -# UDP Flood Scenario IPv4 -- import_playbook: packet-flood.yml - vars: - scenario_send_protocol: UDP - scenario_send_ip_version: 4 - -# UDP Flood Scenario IPv6 -- import_playbook: packet-flood.yml - vars: - scenario_send_protocol: UDP - scenario_send_ip_version: 6 - -# SYN Flood Scenario IPv4 -- import_playbook: packet-flood.yml - vars: - scenario_send_protocol: TCP - scenario_send_ip_version: 4 - -# SYN Flood Scenario IPv6 -- import_playbook: packet-flood.yml - vars: - scenario_send_protocol: TCP - scenario_send_ip_version: 6 diff --git a/playbooks/scenarios/full/discover_full.yml b/playbooks/scenarios/full/discover_full.yml deleted file mode 100644 index 81da20a5..00000000 --- a/playbooks/scenarios/full/discover_full.yml +++ /dev/null @@ -1,17 +0,0 @@ ---- -- hosts: "{{ host }}" - gather_facts: no - tasks: - - name: Read in topology - set_fact: - topo_dict: "{{ lookup('file','{{ topo_file_loc }}') | from_yaml }}" - - - name: Ping inet host - include_tasks: ../test_cases/ping_from.yml - vars: - ip: "{{ topo_dict.hosts['inet'].ip }}" - intf: "{{ item }}" - count: 3 - interval: .5 - pause_time: 0 - loop: "{{ interfaces }}" diff --git a/playbooks/scenarios/full/packet-flood.yml b/playbooks/scenarios/full/packet-flood.yml deleted file mode 100644 index ebb188d5..00000000 --- a/playbooks/scenarios/full/packet-flood.yml +++ /dev/null @@ -1,128 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Simple scenario where packets are sent through 3 devices and only the last -# one will be demonstrating dropped packets ---- -# Sending 150 packets within a second 2x through gateway 1. -# The second set should be dropped else this will fail -- import_playbook: ../test_cases/send_receive.yml - vars: - send_protocol: "{{ scenario_send_protocol | default('UDP') }}" - ip_version: "{{ scenario_send_ip_version | default(4) }}" - topo_dict: "{{ lookup('file','{{ topo_file_loc }}') | from_yaml }}" - switch_name: gateway1 - core_switch_name: core - remote_send_host: Camera1 - remote_rec_host: inet - switch: "{{ topo_dict.switches[switch_name] }}" - send_host: "{{ remote_send_host }}" - rec_host: "{{ remote_rec_host }}" - sender: "{{ topo_dict.hosts[remote_send_host] }}" - receiver: "{{ topo_dict.hosts[remote_rec_host] }}" - switch_egress_intf: eth4 - dst_mac: "{{ sender['switch_mac'] }}" - receiver_log_filename: "{{ send_protocol }}-flood-receiver-{{ rec_host }}-{{ send_protocol }}-1.out" - sender_log_filename: "{{ send_protocol }}-flood-sender-{{ send_host }}-{{ send_protocol }}-1.out" - send_port: "{{ '%s' % range(2000, 10000) | random(seed=now|int + 1) }}" - send_src_port: "{{ '%s' % range(2000, 10000) | random(seed=now|int + 2) }}" - send_packet_count: 150 - min_received_packet_count: 99 - max_received_packet_count: "{{ send_packet_count }}" - send_loops: 2 - send_loop_delay: 5 - cleanup_rest_call: - url: "http://{{ sdn_ip }}:{{ sdn_port }}/gwAttack" - body: - src_ip: "{% if scenario_send_ip_version|int == 4 %}{{ sender.ip }}{% else %}{{ sender.ipv6 }}{% endif %}" - packet_size: 112 - attack_type: "{{ send_protocol }} Flood" - dst_port: "{{ send_port }}" - dst_ip: "{% if scenario_send_ip_version|int == 4 %}{{ receiver.ip }}{% else %}{{ receiver.ipv6 }}{% endif %}" - src_mac: "{{ sender.mac }}" - ok_status: 201 - -# Sending 150 packets within a second 2x through gateway 2. -# The second set should be dropped else this will fail -- import_playbook: ../test_cases/send_receive.yml - vars: - send_protocol: "{{ scenario_send_protocol | default('UDP') }}" - ip_version: "{{ scenario_send_ip_version | default(4) }}" - topo_dict: "{{ lookup('file','{{ topo_file_loc }}') | from_yaml }}" - switch_name: gateway2 - core_switch_name: core - remote_send_host: Camera2 - remote_rec_host: inet - switch: "{{ topo_dict.switches[switch_name] }}" - send_host: "{{ remote_send_host }}" - rec_host: "{{ remote_rec_host }}" - sender: "{{ topo_dict.hosts[remote_send_host] }}" - receiver: "{{ topo_dict.hosts[remote_rec_host] }}" - switch_egress_intf: eth3 - dst_mac: "{{ sender['switch_mac'] }}" - receiver_log_filename: "{{ send_protocol }}-flood-receiver-{{ rec_host }}-{{ send_protocol }}-2.out" - sender_log_filename: "{{ send_protocol }}-flood-sender-{{ send_host }}-{{ send_protocol }}-2.out" - send_port: "{{ '%s' % range(2000, 10000) | random(seed=now|int + 3) }}" - send_src_port: "{{ '%s' % range(2000, 10000) | random(seed=now|int + 4) }}" - send_packet_count: 150 - min_received_packet_count: 99 - max_received_packet_count: "{{ send_packet_count }}" - send_loops: 2 - send_loop_delay: 5 - cleanup_rest_call: - url: "http://{{ sdn_ip }}:{{ sdn_port }}/gwAttack" - body: - src_ip: "{% if scenario_send_ip_version|int == 4 %}{{ sender.ip }}{% else %}{{ sender.ipv6 }}{% endif %}" - packet_size: 112 - attack_type: "{{ send_protocol }} Flood" - dst_port: "{{ send_port }}" - dst_ip: "{% if scenario_send_ip_version|int == 4 %}{{ receiver.ip }}{% else %}{{ receiver.ipv6 }}{% endif %}" - src_mac: "{{ sender.mac }}" - ok_status: 201 - -# Sending 150 packets within a second 2x through gateway 3. -# The second set should be dropped else this will fail -- import_playbook: ../test_cases/send_receive.yml - vars: - send_protocol: "{{ scenario_send_protocol | default('UDP') }}" - ip_version: "{{ scenario_send_ip_version | default(4) }}" - topo_dict: "{{ lookup('file','{{ topo_file_loc }}') | from_yaml }}" - switch_name: gateway3 - core_switch_name: core - remote_send_host: Camera3 - remote_rec_host: inet - switch: "{{ topo_dict.switches[switch_name] }}" - send_host: "{{ remote_send_host }}" - rec_host: "{{ remote_rec_host }}" - sender: "{{ topo_dict.hosts[remote_send_host] }}" - receiver: "{{ topo_dict.hosts[remote_rec_host] }}" - switch_egress_intf: eth3 - dst_mac: "{{ sender['switch_mac'] }}" - receiver_log_filename: "{{ send_protocol }}-flood-receiver-{{ rec_host }}-{{ send_protocol }}-3.out" - sender_log_filename: "{{ send_protocol }}-flood-sender-{{ send_host }}-{{ send_protocol }}-3.out" - send_port: "{{ '%s' % range(2000, 10000) | random(seed=now|int + 5) }}" - send_src_port: "{{ '%s' % range(2000, 10000) | random(seed=now|int + 6) }}" - send_packet_count: 150 - min_received_packet_count: 99 - max_received_packet_count: "{{ send_packet_count }}" - send_loops: 2 - send_loop_delay: 5 - cleanup_rest_call: - url: "http://{{ sdn_ip }}:{{ sdn_port }}/gwAttack" - body: - src_ip: "{% if scenario_send_ip_version|int == 4 %}{{ sender.ip }}{% else %}{{ sender.ipv6 }}{% endif %}" - packet_size: 112 - attack_type: "{{ send_protocol }} Flood" - dst_port: "{{ send_port }}" - dst_ip: "{% if scenario_send_ip_version|int == 4 %}{{ receiver.ip }}{% else %}{{ receiver.ipv6 }}{% endif %}" - src_mac: "{{ sender.mac }}" - ok_status: 201 diff --git a/playbooks/scenarios/gateway/all.yml b/playbooks/scenarios/gateway/all.yml deleted file mode 100644 index dff8d096..00000000 --- a/playbooks/scenarios/gateway/all.yml +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Simple scenario where packets are sent through 3 devices and only the last -# one will be demonstrating dropped packets ---- -# Data Forward scenarios for UDP and IPv4 -- import_playbook: data-forward.yml - vars: - scenario_send_protocol: UDP - scenario_send_ip_version: 4 - -# Data Forward scenarios for UDP and IPv6 -- import_playbook: data-forward.yml - vars: - scenario_send_protocol: UDP - scenario_send_ip_version: 6 - -# Data Forward scenarios for TCP and IPv4 -- import_playbook: data-forward.yml - vars: - scenario_send_protocol: TCP - scenario_send_ip_version: 4 - -# Data Forward scenarios for TCP and IPv6 -- import_playbook: data-forward.yml - vars: - scenario_send_protocol: TCP - scenario_send_ip_version: 6 - -# Data Drop scenarios for UDP and IPv4 -- import_playbook: data-drop.yml - vars: - scenario_send_protocol: UDP - scenario_send_ip_version: 4 - -# Data Drop scenarios for UDP and IPv6 -- import_playbook: data-drop.yml - vars: - scenario_send_protocol: UDP - scenario_send_ip_version: 6 - -# Data Drop scenarios for TCP and IPv4 -- import_playbook: data-drop.yml - vars: - scenario_send_protocol: TCP - scenario_send_ip_version: 4 - -# Data Drop scenarios for TCP and IPv6 -- import_playbook: data-drop.yml - vars: - scenario_send_protocol: TCP - scenario_send_ip_version: 6 - -# Data Inspection scenarios for UDP and IPv4 -- import_playbook: data-inspection.yml - vars: - scenario_send_protocol: UDP - scenario_send_ip_version: 4 - -# Data Inspection scenarios for UDP and IPv6 -- import_playbook: data-inspection.yml - vars: - scenario_send_protocol: UDP - scenario_send_ip_version: 6 - -# Data Inspection scenarios for TCP and IPv4 -- import_playbook: data-inspection.yml - vars: - scenario_send_protocol: TCP - scenario_send_ip_version: 4 - -# Data Inspection scenarios for TCP and IPv6 -- import_playbook: data-inspection.yml - vars: - scenario_send_protocol: TCP - scenario_send_ip_version: 6 diff --git a/playbooks/scenarios/gateway/data-drop.yml b/playbooks/scenarios/gateway/data-drop.yml deleted file mode 100644 index 6278807e..00000000 --- a/playbooks/scenarios/gateway/data-drop.yml +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Simple scenario where packets are sent through 3 devices and only the last -# one will be demonstrating dropped packets ---- - -# Basic Data Drop scenario -- import_playbook: ../general/data_drop_basic.yml - vars: - send_protocol: "{{ scenario_send_protocol | default('UDP') }}" - ip_version: "{{ scenario_send_ip_version | default(4) }}" - topo_dict: "{{ lookup('file','{{ topo_file_loc }}') | from_yaml }}" - switch: "{{ topo_dict.switches.gateway }}" - remote_send_host: host1 - remote_rec_host: host2 - send_host: "{{ remote_send_host }}" - rec_host: "{{ remote_rec_host }}" - send_port: 4321 - sender: "{{ topo_dict.hosts[remote_send_host] }}" - receiver: "{{ topo_dict.hosts[remote_rec_host] }}" - data_forwards: - - url: "http://{{ sdn_ip }}:{{ sdn_port }}/dataForward" - body: - device_id: "{{ switch.id }}" - switch_mac: "{{ switch.mac }}" - dst_mac: "{{ receiver.mac }}" - output_port: 2 - ok_status: 201 - attack: - url: "http://{{ sdn_ip }}:{{ sdn_port }}/gwAttack" - body: - src_ip: "{% if ip_version|int == 4 %}{{ sender.ip }}{% else %}{{ sender.ipv6 }}{% endif %}" - packet_size: 112 - attack_type: "UDP Flood" - dst_port: "{{ send_port }}" - dst_ip: "{% if ip_version|int == 4 %}{{ receiver.ip }}{% else %}{{ receiver.ipv6 }}{% endif %}" - src_mac: "{{ sender.mac }}" - ok_status: 201 diff --git a/playbooks/scenarios/gateway/data-forward.yml b/playbooks/scenarios/gateway/data-forward.yml deleted file mode 100644 index 975eebdb..00000000 --- a/playbooks/scenarios/gateway/data-forward.yml +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Simple scenario where packets are sent through 3 devices and only the last -# one will be demonstrating dropped packets ---- - -# Basic Data Forward scenario for UDP -- import_playbook: ../general/data_forward_basic.yml - vars: - send_protocol: "{{ scenario_send_protocol | default('UDP') }}" - ip_version: "{{ scenario_send_ip_version | default(4) }}" - topo_dict: "{{ lookup('file','{{ topo_file_loc }}') | from_yaml }}" - switch: "{{ topo_dict.switches.gateway }}" - send_host: host1 - rec_host: host2 - sender: "{{ topo_dict.hosts[send_host] }}" - receiver: "{{ topo_dict.hosts[rec_host] }}" - data_forwards: - - url: "http://{{ sdn_ip }}:{{ sdn_port }}/dataForward" - body: - device_id: "{{ switch.id }}" - switch_mac: "{{ switch.mac }}" - dst_mac: "{{ receiver.mac }}" - output_port: 2 - ok_status: 201 diff --git a/playbooks/scenarios/gateway/data-inspection.yml b/playbooks/scenarios/gateway/data-inspection.yml deleted file mode 100644 index be8f1e5f..00000000 --- a/playbooks/scenarios/gateway/data-inspection.yml +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Simple scenario where packets are sent through 3 devices and only the last -# one will be demonstrating dropped packets ---- - -# Basic Data Inspection scenario for UDP -- import_playbook: ../single_switch/data_inspection_basic.yml - vars: - send_protocol: "{{ scenario_send_protocol | default('UDP') }}" - ip_version: "{{ scenario_send_ip_version | default(4) }}" - topo_dict: "{{ lookup('file','{{ topo_file_loc }}') | from_yaml }}" - switch: "{{ topo_dict.switches.gateway }}" - remote_send_host: host1 - remote_rec_host: host2 - send_host: "{{ remote_send_host }}" - rec_host: "{{ remote_rec_host }}" - sender: "{{ topo_dict.hosts[remote_send_host] }}" - receiver: "{{ topo_dict.hosts[remote_rec_host] }}" - ws_calls: - - url: "http://{{ sdn_ip }}:{{ sdn_port }}/dataForward" - body: - device_id: "{{ switch.id }}" - switch_mac: "{{ switch.mac }}" - dst_mac: "{{ receiver.mac }}" - output_port: 2 - ok_status: 201 - data_inspections: - - url: "http://{{ sdn_ip }}:{{ sdn_port }}/dataInspection" - body: - device_id: "{{ switch.id }}" - switch_mac: "{{ switch.mac }}" - device_mac: "{{ sender.mac }}" - ok_status: 201 - sr1_rec_int_hops: 1 diff --git a/playbooks/scenarios/general/drop_report_simple.yml b/playbooks/scenarios/general/drop_report_simple.yml index f6c32efa..dd3d4015 100644 --- a/playbooks/scenarios/general/drop_report_simple.yml +++ b/playbooks/scenarios/general/drop_report_simple.yml @@ -85,7 +85,7 @@ command: "{{ receive_cmd }}" register: receive_out changed_when: receive_out is not failed - retries: 3 + retries: 6 delay: 10 until: receive_out.stdout | length > 5 and receive_out.stdout | from_json | length == send_ports | length diff --git a/playbooks/siddhi/templates/convert_trpt.siddhi.j2 b/playbooks/siddhi/templates/convert_trpt.siddhi.j2 index 468b7083..f2a33720 100644 --- a/playbooks/siddhi/templates/convert_trpt.siddhi.j2 +++ b/playbooks/siddhi/templates/convert_trpt.siddhi.j2 @@ -13,7 +13,7 @@ * limitations under the License. */ -@app:name('UDP-Source-TRPT') +@App:name('UDPSourceTRPT') @source(type='udp', listen.port='{{ telem_rpt_port }}', @map(type='p4-trpt', @attributes(in_type='telemRptHdr.inType', full_json='jsonString'))) define stream trptUdpStream (in_type int, full_json object); diff --git a/playbooks/siddhi/templates/simple_ddos_clear_drop.siddhi.j2 b/playbooks/siddhi/templates/simple_ddos_clear_drop.siddhi.j2 index c4117754..eb1a7c6c 100644 --- a/playbooks/siddhi/templates/simple_ddos_clear_drop.siddhi.j2 +++ b/playbooks/siddhi/templates/simple_ddos_clear_drop.siddhi.j2 @@ -13,7 +13,7 @@ * limitations under the License. */ -@app:name('Kafka-Source-Drop-JSON') +@App:name('KafkaSourceDropJSON') @source(type='kafka', topic.list='{{ kafka_trpt_drop_topic }}', bootstrap.servers='{{ kafka_host_port }}', group.id='test', threading.option='single.thread', diff --git a/playbooks/siddhi/templates/simple_ddos_detection.siddhi.j2 b/playbooks/siddhi/templates/simple_ddos_detection.siddhi.j2 index 54b97080..072cbe12 100644 --- a/playbooks/siddhi/templates/simple_ddos_detection.siddhi.j2 +++ b/playbooks/siddhi/templates/simple_ddos_detection.siddhi.j2 @@ -13,7 +13,7 @@ * limitations under the License. */ -@app:name('Kafka-Source-Packet-JSON') +@App:name('KafkaSourcePacketJSON') @source(type='kafka', topic.list='{{ kafka_trpt_pkt_topic }}', bootstrap.servers='{{ kafka_host_port }}', group.id='test', threading.option='single.thread', diff --git a/playbooks/tofino/setup_nodes-full.yml b/playbooks/tofino/setup_nodes-full.yml deleted file mode 100644 index 4a8b385f..00000000 --- a/playbooks/tofino/setup_nodes-full.yml +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -- import_playbook: ../general/setup_source.yml - vars: - host_val: - - hosts - trans_sec_source_dir: "{{ orch_trans_sec_dir }}" - python_unit_tests: false - -# Create virtual interfaces on switches -- import_playbook: setup_virt_eth.yml - vars: - host_val: switches - remote_sde_dir: "{{ sde_dir }}" - -# Setup GRE Tunnels -- import_playbook: tunnel/setup_gre_tunnels.yml - -# TODO/FIXME - Call setup_tofino_switch playbook so it can run in parallel - -# Start tofino-model and tofino-switchd on the core switch -- import_playbook: setup_tofino_switch.yml - vars: - switches: - - core - host_val: core - p4_prog: core.p4 - topo_dict: "{{ lookup('file','{{ topo_file_loc }}') | from_yaml }}" - tunnels: "{{ topo_dict.switches['core'].tunnels }}" - load_tofino_p4rt: False - -# Start tofino-model and tofino-switchd on the aggregate switch -- import_playbook: setup_tofino_switch.yml - vars: - switches: - - aggregate - host_val: aggregate - p4_prog: aggregate.p4 - topo_dict: "{{ lookup('file','{{ topo_file_loc }}') | from_yaml }}" - tunnels: "{{ topo_dict.switches['aggregate'].tunnels }}" - -# Start tofino-model and tofino-switchd on gateway1 -- import_playbook: setup_tofino_switch.yml - vars: - switches: - - gateway1 - - gateway2 - - gateway3 - p4_prog: gateway.p4 - -# Start SDN Controller -- import_playbook: ../general/start_service.yml - vars: - host_val: controller - service_name: tps-tofino-sdn - scenario_name: full - local_srvc_script_tmplt_file: "{{ orch_trans_sec_dir }}/playbooks/general/templates/sdn_controller.sh.j2" - port_to_wait: "{{ sdn_port }}" - wait_timeout: 30 - load_p4: False - -# Start AE -- import_playbook: ../general/start_service.yml - vars: - host_val: ae - service_name: tps-tofino-ae - local_srvc_script_tmplt_file: "{{ orch_trans_sec_dir }}/playbooks/general/templates/ae_service.sh.j2" - srvc_type: SIMPLE - sdn_url: "http://{{ sdn_ip }}:{{ sdn_port }}" - monitor_intf: "{{ ae_monitor_intf }}" diff --git a/playbooks/tofino/setup_orchestrator-full.yml b/playbooks/tofino/setup_orchestrator-full.yml deleted file mode 100644 index e09d1215..00000000 --- a/playbooks/tofino/setup_orchestrator-full.yml +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright (c) 2020 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -- import_playbook: setup_orchestrator.yml - vars: - topology_template: templates/topology_template-full.yaml.j2 diff --git a/playbooks/tofino/setup_orchestrator.yml b/playbooks/tofino/setup_orchestrator.yml index 5ccbc7aa..96d327e7 100644 --- a/playbooks/tofino/setup_orchestrator.yml +++ b/playbooks/tofino/setup_orchestrator.yml @@ -20,6 +20,12 @@ vars: local_topology: /tmp/tofino-topology.yaml tasks: + - name: Create {{ remote_scripts_dir }} + become: yes + file: + path: "{{ remote_scripts_dir }}" + state: directory + - name: Apply template to topology configuration file {{ topology_template }} become: yes template: diff --git a/snaps-hcp/OpenDistroElasticSearch/Readme.md b/snaps-hcp/OpenDistroElasticSearch/Readme.md deleted file mode 100644 index b71cc0d7..00000000 --- a/snaps-hcp/OpenDistroElasticSearch/Readme.md +++ /dev/null @@ -1,4 +0,0 @@ -# Open Distro Elasticsearch Guides - -- To visualize and injest data use the [Data Mapping and Parsing Guide](/udp_data_mapping_and_parsing_pipeline.md) -- To detect DDOS attack and mitigate it use the [DDOS Detection and Mitigation Guide](ddos_detection_and_mitigation_pipeline.md) diff --git a/snaps-hcp/OpenDistroElasticSearch/anomaly_detector_config.json b/snaps-hcp/OpenDistroElasticSearch/anomaly_detector_config.json deleted file mode 100644 index 7312cc76..00000000 --- a/snaps-hcp/OpenDistroElasticSearch/anomaly_detector_config.json +++ /dev/null @@ -1,80 +0,0 @@ -{ - "name": "ddos-api-anomaly-detector", - "description": "DDOS API Anomaly Detector", - "time_field": "timestamp", - "indices": [ - "packets-*" - ], - "feature_attributes": [ - { - "feature_name": "UDP2DstPortValueCount", - "feature_enabled": true, - "aggregation_query": { - "UDP2DstPortValueCount": { - "value_count": { - "field": "ts.UDP2DstPort" - } - } - } - }, - { - "feature_name": "IPVDstAddrValueCount", - "feature_enabled": true, - "aggregation_query": { - "IPVDstAddrValueCount": { - "value_count": { - "field": "ts.IPvDestAddr" - } - } - } - }, - { - "feature_name": "OriginatingMacAddressValueCount", - "feature_enabled": true, - "aggregation_query": { - "OriginatingMacAddressValueCount": { - "value_count": { - "field": "ts.INTMetadataSourceMetadataOriginatingMac" - } - } - } - } - ], - "filter_query": { - "bool": { - "filter": [ - { - "term": { - "ts.IPVersion": { - "value": "4", - "boost": 1 - } - } - }, - { - "wildcard": { - "ts.INTMetadataSourceMetadataOriginatingMac": { - "wildcard": "*000000000101*", - "boost": 1 - } - } - } - ], - "adjust_pure_negative": true, - "boost": 1 - } - }, - "detection_interval": { - "period": { - "interval": 1, - "unit": "Minutes" - } - }, - "window_delay": { - "period": { - "interval": 1, - "unit": "Minutes" - } - }, - "shingle_size" : 1 -} diff --git a/snaps-hcp/OpenDistroElasticSearch/data_mapping.json b/snaps-hcp/OpenDistroElasticSearch/data_mapping.json deleted file mode 100644 index c919b6c7..00000000 --- a/snaps-hcp/OpenDistroElasticSearch/data_mapping.json +++ /dev/null @@ -1,264 +0,0 @@ -{ - "index_patterns": [ - "packets-*" - ], - "settings": { - "index.number_of_replicas": "0", - "index.max_script_fields": "200", - "index.default_pipeline": "ts_parsing" - }, - "mappings": { - "dynamic": "false", - "properties": { - "timestamp": { - "type": "date" - }, - "ts": { - "properties": { - "rawData": { - "type": "keyword" - }, - "tsRawDataSize": { - "type": "keyword" - }, - "udpLayersSize": { - "type": "keyword" - }, - "telemetryReport": { - "type": "keyword" - }, - "telemetryReportVersion": { - "type": "keyword" - }, - "telemetryHwdId": { - "type": "keyword" - }, - "telemetrySequenceNumber": { - "type": "keyword" - }, - "telemetryNodeId": { - "type": "keyword" - }, - "telemetryReportType": { - "type": "keyword" - }, - "telemetryInType": { - "type": "keyword" - }, - "telemetryReportLength": { - "type": "keyword" - }, - "telemetryMDLength": { - "type": "keyword" - }, - "telemetryD": { - "type": "keyword" - }, - "telemetryQ": { - "type": "keyword" - }, - "telemetryF": { - "type": "keyword" - }, - "telemetryI": { - "type": "keyword" - }, - "telemetryRsvd": { - "type": "keyword" - }, - "telemetryRepMdBits": { - "type": "keyword" - }, - "telemetryDomainSpecificID": { - "type": "keyword" - }, - "telemetryDsMdBits": { - "type": "keyword" - }, - "telemetryDsMdStatus": { - "type": "keyword" - }, - "telemetryVarOpt": { - "type": "keyword" - }, - "data": { - "type": "keyword" - }, - "ethernetHeaderDestMac": { - "type": "keyword" - }, - "ethernetHeaderSrcMac": { - "type": "keyword" - }, - "ethernetHeaderEtherType": { - "type": "keyword" - }, - "IPvData": { - "type": "keyword" - }, - "isIpv6": { - "type": "keyword" - }, - "isIpv4": { - "type": "keyword" - }, - "IPVersion": { - "type": "keyword" - }, - "IPvClass": { - "type": "keyword" - }, - "IPvFlow": { - "type": "keyword" - }, - "IPvPayloadLength": { - "type": "keyword" - }, - "IPvNextHdrProto": { - "type": "keyword" - }, - "IPvNextHopLimit": { - "type": "keyword" - }, - "IPvSrcAddr": { - "type": "keyword" - }, - "IPvDestAddr": { - "type": "keyword" - }, - "UDGPData": { - "type": "keyword" - }, - "UDGSrcPort": { - "type": "keyword" - }, - "UDGDstPort": { - "type": "keyword" - }, - "UDGLength": { - "type": "keyword" - }, - "UDGChecksum": { - "type": "keyword" - }, - "InBandNetworkTelemetryData": { - "type": "keyword" - }, - "UDPIntShimHeaderData": { - "type": "keyword" - }, - "UDPIntShimHeaderType": { - "type": "keyword" - }, - "UDPIntShimHeaderNPT": { - "type": "keyword" - }, - "UDPIntShimHeaderRes1": { - "type": "keyword" - }, - "UDPIntShimHeaderLength": { - "type": "keyword" - }, - "UDPIntShimHeaderRes2": { - "type": "keyword" - }, - "UDPIntShimHeaderNextProto": { - "type": "keyword" - }, - "INTMetadataHeaderData": { - "type": "keyword" - }, - "UDPIntMetaDataHeaderVersion": { - "type": "keyword" - }, - "UDPIntMetaDataHeaderRes": { - "type": "keyword" - }, - "UDPIntMetaDataHeaderD": { - "type": "keyword" - }, - "UDPIntMetaDataHeaderE": { - "type": "keyword" - }, - "UDPIntMetaDataHeaderM": { - "type": "keyword" - }, - "UDPIntMetaDataHeaderPerhopMetadataLength": { - "type": "keyword" - }, - "UDPIntMetaDataHeaderRemainingHopCnt": { - "type": "keyword" - }, - "UDPIntMetaDataHeaderInstructionBitmap": { - "type": "keyword" - }, - "UDPIntMetaDataHeaderDomainSpecificID": { - "type": "keyword" - }, - "UDPIntMetaDataHeaderDSInstruction": { - "type": "keyword" - }, - "UDPIntMetaDataHeaderDSFlags": { - "type": "keyword" - }, - "INTMetadataStackData": { - "type": "keyword" - }, - "INTMetadataStackSwitchID": { - "type": "keyword" - }, - "INTMetadataSourceMetadata": { - "type": "keyword" - }, - "INTMetadataSourceMetadataSwitchId": { - "type": "keyword" - }, - "INTMetadataSourceMetadataOriginatingMac": { - "type": "keyword" - }, - "INTMetadataSourceMetadataReserved": { - "type": "keyword" - }, - "UDP2Data": { - "type": "keyword" - }, - "UDP2SrcPort": { - "type": "keyword" - }, - "UDP2DstPort": { - "type": "keyword" - }, - "UDP2Length": { - "type": "keyword" - }, - "UDP2Checksum": { - "type": "keyword" - }, - "dataII": { - "type": "keyword" - } - } - }, - "layers": { - "properties": { - "data_raw": { - "type": "keyword" - }, - "data": { - "properties": { - "data_data_data_raw": { - "type": "keyword" - }, - "data_data_data": { - "type": "keyword" - }, - "data_data_len": { - "type": "keyword" - } - } - } - } - } - } - } -} diff --git a/snaps-hcp/OpenDistroElasticSearch/ddos_detection_and_mitigation_pipeline.md b/snaps-hcp/OpenDistroElasticSearch/ddos_detection_and_mitigation_pipeline.md deleted file mode 100644 index ebf8341f..00000000 --- a/snaps-hcp/OpenDistroElasticSearch/ddos_detection_and_mitigation_pipeline.md +++ /dev/null @@ -1,360 +0,0 @@ -# DDOS Detection and Mitigation Pipeline ( UDPv4 ) using Open Distro Elasticsearch - -- To add a DDOS Detection pipeline you need to add a monitor and an associated trigger to detect the DDOS attack. -- To add a DDOS Mitigation pipeline you need add an associated action (custom webhook) with associated trigger. - -The same can be done using the Kibana UI or programmatically using the [Anomaloy Detection API]( https://opendistro.github.io/for-elasticsearch-docs/docs/ad/api/ ) and [Alerting API]( https://opendistro.github.io/for-elasticsearch-docs/docs/alerting/api/ ) - -The below guides talks about using the Anomaloy Detection API and Alerting API. - -> **_NOTE:_** -The guide is specific to DDOS use case. Please change the Anomaloy Detectors, Monitors, Triggers and Actions according to your use case. -[Official Guide](https://opendistro.github.io/for-elasticsearch-docs/) - -## Add Anomaloy Detector -To create an Anomaloy Detector using the Kibana UI please look at [Official Guide](https://opendistro.github.io/for-elasticsearch-docs/docs/ad/) for more details. - -Use below anomaly detection operations to programmatically create and manage detectors. - -### Step 1: Create Anomaloy Detector - -- Push it -``` -curl -H 'Content-Type: application/json' -XPOST 'http://localhost:9200/_opendistro/_anomaly_detection/detectors' -d @anomaly_detector_config.json -``` -- Expected Response -``` -{ - "_id" : "UDkwkXUBenaD-0rLhn9b", - "_version" : 1, - "_seq_no" : 5, - "_primary_term" : 1, - "anomaly_detector" : { - "name" : "ddos-anomaly-detector", - "description" : "DDOS Anomaly Detector", - "time_field" : "timestamp", - "indices" : [ - "packets-*" - ], - "filter_query" : { - "bool" : { - "filter" : [ - { - "term" : { - "ts.IPVersion" : { - "value" : "4", - "boost" : 1.0 - } - } - }, - { - "wildcard" : { - "ts.INTMetadataSourceMetadataOriginatingMac" : { - "wildcard" : "*000000000101*", - "boost" : 1.0 - } - } - } - ], - "adjust_pure_negative" : true, - "boost" : 1.0 - } - }, - "detection_interval" : { - "period" : { - "interval" : 1, - "unit" : "Minutes" - } - }, - "window_delay" : { - "period" : { - "interval" : 1, - "unit" : "Minutes" - } - }, - "shingle_size" : 1, - "schema_version" : 0, - "feature_attributes" : [ - { - "feature_id" : "STkwkXUBenaD-0rLhn9Z", - "feature_name" : "UDP2DstPortValueCount", - "feature_enabled" : true, - "aggregation_query" : { - "UDP2DstPortValueCount" : { - "value_count" : { - "field" : "ts.UDP2DstPort" - } - } - } - }, - { - "feature_id" : "SjkwkXUBenaD-0rLhn9Z", - "feature_name" : "IPVDstAddrValueCount", - "feature_enabled" : true, - "aggregation_query" : { - "IPVDstAddrValueCount" : { - "value_count" : { - "field" : "ts.IPvDestAddr" - } - } - } - }, - { - "feature_id" : "SzkwkXUBenaD-0rLhn9Z", - "feature_name" : "OriginatingMacAddressValueCount", - "feature_enabled" : true, - "aggregation_query" : { - "OriginatingMacAddressValueCount" : { - "value_count" : { - "field" : "ts.INTMetadataSourceMetadataOriginatingMac" - } - } - } - } - ] - } -} -``` -- Verify it worked - -The following GET call with appropriate _id can be made to check the newly created anomaly detector. The _id can be found in reponse of Step 1 - -``` -curl -H 'Content-Type: application/json' -XPOST 'http://localhost:9200/_opendistro/_anomaly_detection/detectors/{_id} -``` - -- Start Anomaloy Detector - -``` -curl -H 'Content-Type: application/json' -XPOST 'http://localhost:9200/_opendistro/_anomaly_detection/detectors/{_id}/_start -``` - -### Step 2: Create Monitor - -A monitor is a job that runs on a defined schedule and queries Elasticsearch. The results of these queries are then used as input for one or more triggers. - -- Create custom SDN Webhook Alert Action to be associated with the Monitor - -``` -curl -H 'Content-Type: application/json' -XPOST 'http://localhost:9200/_opendistro/_alerting/destinations -d -'{ - "type": "custom_webhook", - "name": "SDN_Webhook", - "custom_webhook": { - "path": "/aggAttack", - "header_params": { - "Content-Type": "application/json" - }, - "scheme": "HTTP", - "port": 9998, - "host": "tps.sdn.org" - } -}' -``` -- Expected Response -``` -{ - "_id" : "jjlCkXUBenaD-0rLO4Cg", - "_version" : 1, - "_seq_no" : 16, - "_primary_term" : 1, - "destination" : { - "type" : "custom_webhook", - "name" : "SDN_Webhook", - "schema_version" : 2, - "last_update_time" : 1604459838368, - "custom_webhook" : { - "path" : "/aggAttack", - "header_params" : { - "Content-Type" : "application/json" - }, - "password" : null, - "scheme" : "HTTP", - "port" : 9998, - "query_params" : { }, - "host" : "tps.sdn.org", - "url" : null, - "username" : null - } - } -} -``` - -> **_NOTE:_** -> Edit the /etc/hosts file on analytics instance to add the ubuntu private IP for dns name resolution to tps.sdn.org or any other preferred dns name. The DNS name added in hosts file should correspond to host properties in above payload. -> The _id from response will be utilized when creating the associated Monitor. - -- Create Monitor - -``` -curl -H 'Content-Type: application/json' -XPOST 'http://localhost:9200/ _opendistro/_alerting/monitors -d -'{ - "type": "monitor", - "name": "DDOS-Monitor", - "enabled": true, - "schedule": { - "period": { - "interval": 1, - "unit": "MINUTES" - } - }, - "inputs": [{ - "search": { - "indices": ["packets-*"], - "query": { - "size": 0, - "query": { - "bool": { - "filter": [ - { - "range": { - "timestamp": { - "from": "{{period_end}}||-1m", - "to": "{{period_end}}", - "include_lower": true, - "include_upper": true, - "format": "epoch_millis", - "boost": 1 - } - } - }, - { - "term": { - "ts.IPVersion": { - "value": "4", - "boost": 1 - } - } - }, - { - "wildcard": { - "ts.IPvDestAddr": { - "wildcard": "*c0a8010a*", - "boost": 1 - } - } - }, - { - "wildcard": { - "ts.UDP2DstPort": { - "wildcard": "*5792*", - "boost": 1 - } - } - } - ], - "adjust_pure_negative": true, - "boost": 1 - } - }, - "_source": { - "includes": [ - "ts.UDP2DstPort", - "ts.IPvDestAddr", - "ts.INTMetadataSourceMetadataOriginatingMac", - "ts.IPv4SrcIP", - "ts.IPv4DestIP", - "ts.IPv6SrcIP", - "ts.IPv6DestIP" - ], - "excludes": [] - }, - "aggregations": { - "UDP2DstPortValueCount": { - "value_count": { - "field": "ts.UDP2DstPort" - } - }, - "IPvDestAddrValueCount": { - "value_count": { - "field": "ts.IPvDestAddr" - } - }, - "INTMetadataSourceMetadataOriginatingMacValueCount": { - "value_count": { - "field": "ts.INTMetadataSourceMetadataOriginatingMac" - } - }, - "OriginatingMacAddrUDPv4": { - "terms": { - "field": "ts.INTMetadataSourceMetadataOriginatingMac", - "size": 10, - "min_doc_count": 1, - "shard_min_doc_count": 0, - "show_term_doc_count_error": false, - "order": [ - { - "_count": "desc" - }, - { - "_key": "asc" - } - ] - } - }, - "IPv4SrcIP": { - "terms": { - "field": "ts.IPv4SrcIP", - "size": 10, - "min_doc_count": 1, - "shard_min_doc_count": 0, - "show_term_doc_count_error": false, - "order": [ - { - "_count": "desc" - }, - { - "_key": "asc" - } - ] - } - }, - "IPv4DestIP": { - "terms": { - "field": "ts.IPv4DestIP", - "size": 10, - "min_doc_count": 1, - "shard_min_doc_count": 0, - "show_term_doc_count_error": false, - "order": [ - { - "_count": "desc" - }, - { - "_key": "asc" - } - ] - } - } - } - } - } - }], - "triggers": [{ - "name": "DDOS-Trigger", - "severity": "1", - "condition": { - "script": { - "source": "ctx.results[0].aggregations.UDP2DstPortValueCount.value >= 1000 && ctx.results[0].aggregations.IPvDestAddrValueCount.value >= 1000 && ctx.results[0].aggregations.INTMetadataSourceMetadataOriginatingMacValueCount.value >= 1000", - "lang": "painless" - } - }, - "actions": [{ - "name": "Sdn-WebHook-Action", - "destination_id": "jjlCkXUBenaD-0rLO4Cg", - "message_template": { - "source":"{\"event_action\":\"trigger\",\"payload\":{ \"dst_ip\":\"{{_source.ts.IPv4DestIP}}\",\"dst_port\":\"{{_source.ts.UDP2DstPort}}\",\"src_ip\":\"{{_source.ts.IPv4SrcIP}}\",\"src_mac\":\"{{_source.ts.INTMetadataSourceMetadataOriginatingMac}}\"}}", - "lang": "mustache", - "options": { - "content_type": "application/json" - } - }, - "throttle_enabled": false - }] - }] -}' -``` -> **_NOTE:_** -> The message_template property is specific to each brought up instance in tofino build environment and changes accordingly. - -Once the associated monitor, trigger and action are configured you can test by sending packets specific to your testing scenario. diff --git a/snaps-hcp/OpenDistroElasticSearch/packet_template-7.x.sh b/snaps-hcp/OpenDistroElasticSearch/packet_template-7.x.sh deleted file mode 100755 index c6c373d4..00000000 --- a/snaps-hcp/OpenDistroElasticSearch/packet_template-7.x.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash - -if [[ $# -ne 1 ]] ; then - echo "packet_template.sh : Missing ElasticSearch server address .... " - exit -fi - -curl -H 'Content-Type: application/json' -XPUT 'http://'$1'/_template/packets' -d @data_mapping.json diff --git a/snaps-hcp/OpenDistroElasticSearch/udp_data_mapping_and_parsing_pipeline.md b/snaps-hcp/OpenDistroElasticSearch/udp_data_mapping_and_parsing_pipeline.md deleted file mode 100644 index 624adf15..00000000 --- a/snaps-hcp/OpenDistroElasticSearch/udp_data_mapping_and_parsing_pipeline.md +++ /dev/null @@ -1,34 +0,0 @@ -# UDP Data Mapping and Parsing Guide for Open Distro Elasticsearch - -- Default configured ElasticSearch instance runs on port 9200. -- Default configured Kibana instance runs on port 5601. - -## Adding UDP data templates to ElasticSearch for incoming packets -- To manually add UDP data templates for incoming UDP packets please run packet_template-7.x.sh script. The script takes ElasticSearch address and port number as first argument. -- Push it -``` - ./packet_template-7.x.sh localhost:9200 -``` -If you want to change the data templates for your packets, just revise the udp_data_parsing.json to your liking and re-run the packet_template-7.x.sh script.You will have to delete your existing packet indexing template, if any, before creating a new one. - -## Adding UDP data parsing to ElasticSearch for incoming packets -- To manually add UDP data parsing using Painless API for incoming UDP packets please run udp_data_parsing_pipeline.sh script. The script takes ElasticSearch address and port number as first argument. -- Push it -``` - ./udp_data_parsing_pipeline.sh localhost:9200 -``` - -If you want to change the data parsing pipeline for your packets, just revise the udp_data_parsing_pipeline.json to your liking and re-run the udp_data_parsing_pipeline.sh script. You will have to delete your existing parsing pipeline, if any, before creating a new one. - -## Commands to test added UDP templates and parsing pipeline for incoming packets -- To verify added UDP data template run the following ... - -``` - curl -H 'Content-Type: application/json' -XGET 'http://localhost:9200/_template/packets' -``` - -- To verify added UDP data parsing pipeline run the following ... - -``` - curl -H 'Content-Type: application/json' -XGET 'http://localhost:9200/_ingest/pipeline/ts_parsing' -``` diff --git a/snaps-hcp/OpenDistroElasticSearch/udp_data_parsing.json b/snaps-hcp/OpenDistroElasticSearch/udp_data_parsing.json deleted file mode 100644 index 6f1ba17a..00000000 --- a/snaps-hcp/OpenDistroElasticSearch/udp_data_parsing.json +++ /dev/null @@ -1,1246 +0,0 @@ -{ - "description": "Transparent Security UDP v4 & v6 data parsing pipeline", - "processors": [ - { - "set" : { - "field" : "ts.rawData", - "value" : "{{layers.data.data_data_data}}" - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": "ctx.ts.telemetryReport =ctx.ts.rawData.substring(0, 48).replace(':','')" - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String hardwareIdStr = ctx.layers.data.data_data_data.replace(':','').substring(1, 3); - BigInteger hwdIdBigInt = new BigInteger(hardwareIdStr, 16); - BigInteger bitwiseAnd = new BigInteger('FC', 16); - BigInteger extractedHardwareId = hwdIdBigInt.and(bitwiseAnd); - String hwdIdStr = extractedHardwareId.toString(16); - ctx.ts.telemetryHwdId = hwdIdStr; - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String sequenceNumberPartStrI = tsRawData.substring(2, 3); - String sequenceNumberPartStrII = tsRawData.substring(3, 8); - String sequenceNumberPartStr = tsRawData.substring(2, 8); - BigInteger sequenceNumberBigInt = new BigInteger(sequenceNumberPartStr, 16); - BigInteger bitwiseAnd = new BigInteger('7FFFFFF', 16); - BigInteger updatedSequenceNumber = sequenceNumberBigInt.and(bitwiseAnd); - // String sequenceNumberHexStr = updatedSequenceNumber.toString(16); - ctx.ts.telemetrySequenceNumber = updatedSequenceNumber.intValue(); - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String nodeIdStr = tsRawData.substring(8, 16); - BigInteger nodeIdBigInt = new BigInteger(nodeIdStr, 16); - ctx.ts.telemetryNodeId =nodeIdBigInt.intValue(); - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String repTypeStr = tsRawData.substring(16, 17); - BigInteger repTypeBigInt = new BigInteger(repTypeStr, 16); - ctx.ts.telemetryReportType =repTypeBigInt.intValue(); - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String inTypeStr = tsRawData.substring(17, 18); - BigInteger inTypeBigInt = new BigInteger(inTypeStr, 16); - ctx.ts.telemetryInType = inTypeBigInt.intValue(); - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String reportLengthStr = tsRawData.substring(18, 20); - BigInteger reportLengthBigInt = new BigInteger(reportLengthStr, 16); - ctx.ts.telemetryReportLength = reportLengthBigInt.intValue(); - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String mdLengthStr = tsRawData.substring(20, 22); - BigInteger mdLengthBigInt = new BigInteger(mdLengthStr, 16); - ctx.ts.telemetryMDLength = mdLengthBigInt.intValue(); - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String dStr = tsRawData.substring(22, 23); - BigInteger dBigInt = new BigInteger(dStr, 16); - BigInteger bitwiseAnd = new BigInteger('8', 16); - BigInteger dRes = dBigInt.and(bitwiseAnd); - ctx.ts.telemetryD = dRes.intValue(); - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String qStr = tsRawData.substring(22, 23); - BigInteger qBigInt = new BigInteger(qStr, 16); - BigInteger bitwiseAnd = new BigInteger('4', 16); - BigInteger qRes = qBigInt.and(bitwiseAnd); - ctx.ts.telemetryQ = qRes.intValue(); - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String fStr = tsRawData.substring(22, 23); - BigInteger fBigInt = new BigInteger(fStr, 16); - BigInteger bitwiseAnd = new BigInteger('2', 16); - BigInteger fRes = fBigInt.and(bitwiseAnd); - ctx.ts.telemetryF = fRes.intValue(); - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String IStr = tsRawData.substring(22, 23); - BigInteger IBigInt = new BigInteger(IStr, 16); - BigInteger bitwiseAnd = new BigInteger('1', 16); - BigInteger IRes = IBigInt.and(bitwiseAnd); - ctx.ts.telemetryI = IRes.intValue(); - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String rsvdStr = tsRawData.substring(23, 24); - BigInteger rsvdBigInt = new BigInteger(rsvdStr, 16); - ctx.ts.telemetryRsvd = rsvdBigInt.intValue(); - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String repMdBitsStr = tsRawData.substring(24, 28); - BigInteger repMdBitsBigInt = new BigInteger(repMdBitsStr, 16); - ctx.ts.telemetryRepMdBits = repMdBitsBigInt.intValue(); - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String domainSpecificIDStr = tsRawData.substring(28, 32); - BigInteger domainSpecificIDBigInt = new BigInteger(domainSpecificIDStr, 16); - ctx.ts.telemetryDomainSpecificID = domainSpecificIDBigInt.intValue(); - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String dsMdBitsStr = tsRawData.substring(32, 36); - BigInteger dsMdBitsStrBigInt = new BigInteger(dsMdBitsStr, 16); - ctx.ts.telemetryDsMdBits = dsMdBitsStrBigInt.intValue(); - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String dsMdStatusStr = tsRawData.substring(36, 40); - BigInteger dsMdStatusBigInt = new BigInteger(dsMdStatusStr, 16); - ctx.ts.telemetryDsMdStatus = dsMdStatusBigInt.intValue(); - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String varOptStr = tsRawData.substring(40, 48); - BigInteger varOptBigInt = new BigInteger(varOptStr, 16); - ctx.ts.telemetryVarOpt = varOptBigInt.intValue(); - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.isIPv6 = true; - } else - { - ctx.ts.isIPv6 = false; - } - - - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.isIPv4 = true; - } else - { - ctx.ts.isIPv4 = false; - } - - - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.data = tsRawData.substring(48, 252); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.data = tsRawData.substring(48, 292); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - ctx.ts.ethernetHeader = tsRawData.substring(48, 76); - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - ctx.ts.ethernetHeaderDestMac = tsRawData.substring(48, 60); - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - ctx.ts.ethernetHeaderSrcMac = tsRawData.substring(60, 72); - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - ctx.ts.ethernetHeaderEtherType = tsRawData.substring(72, 76); - - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.IPvData = tsRawData.substring(76, 116); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.IPvData = tsRawData.substring(76, 156); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.IPVersion = tsRawData.substring(76, 77); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.IPVersion = tsRawData.substring(76, 77); - } - """ - } - }, - { - "script" : { - "lang" : "painless", - "ignore_failure" : true, - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String IPv4SrcIPHexString = tsRawData.substring(100, 108); - String IPv4SrcIP = ""; - for(int i = 0; i < IPv4SrcIPHexString.length(); i = i + 2) { - IPv4SrcIP = IPv4SrcIP + Integer.valueOf(IPv4SrcIPHexString.substring(i, i+2), 16) + "."; - } - IPv4SrcIP = IPv4SrcIP.substring(0, IPv4SrcIP.length()-1); - ctx.ts.IPv4SrcIP = IPv4SrcIP; - } - if( etherType == '86dd' && ipVersion == '6') - { - String IPv6SrcIPHexString = tsRawData.substring(92,124); - String IPv6SrcIP = ""; - for(int i = 0; i < IPv6SrcIPHexString.length(); i = i + 2) { - IPv6SrcIP = IPv6SrcIP + Integer.valueOf(IPv6SrcIPHexString.substring(i, i+2), 16) + "."; - } - IPv6SrcIP = IPv6SrcIP.substring(0, IPv6SrcIP.length()-1); - ctx.ts.IPv6SrcIP = IPv6SrcIP; - } - """ - } - }, - { - "script" : { - "lang" : "painless", - "ignore_failure" : true, - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String IPv4DestIPHexString = tsRawData.substring(108, 116); - String IPv4DestIP = ""; - for(int i = 0; i < IPv4DestIPHexString.length(); i = i + 2) { - IPv4DestIP = IPv4DestIP + Integer.valueOf(IPv4DestIPHexString.substring(i, i+2), 16) + "."; - } - IPv4DestIP = IPv4DestIP.substring(0, IPv4DestIP.length()-1); - ctx.ts.IPv4DestIP = IPv4DestIP; - } - if( etherType == '86dd' && ipVersion == '6') - { - String IPv6DestIPHexString = tsRawData.substring(124,156); - String IPv6DestIP = ""; - for(int i = 0; i < IPv6DestIPHexString.length(); i = i + 2) { - IPv6DestIP = IPv6DestIP + Integer.valueOf(IPv6DestIPHexString.substring(i, i+2), 16) + "."; - } - IPv6DestIP = IPv6DestIP.substring(0, IPv6DestIP.length()-1); - ctx.ts.IPv6DestIP = IPv6DestIP; - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDGPData = tsRawData.substring(116, 132); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDGPData = tsRawData.substring(156, 172); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDGSrcPortStr = tsRawData.substring(116, 120); - BigInteger UDGSrcPortBigInt = new BigInteger(UDGSrcPortStr, 16); - ctx.ts.UDGSrcPort = UDGSrcPortBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDGSrcPortStr = tsRawData.substring(156, 160); - BigInteger UDGSrcPortBigInt = new BigInteger(UDGSrcPortStr, 16); - ctx.ts.UDGSrcPort = UDGSrcPortBigInt.intValue(); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDGDstPortStr = tsRawData.substring(120, 124); - BigInteger UDGDstPortBigInt = new BigInteger(UDGDstPortStr, 16); - ctx.ts.UDGDstPort = UDGDstPortBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDGDstPortStr = tsRawData.substring(160, 164); - BigInteger UDGDstPortBigInt = new BigInteger(UDGDstPortStr, 16); - ctx.ts.UDGDstPort = UDGDstPortBigInt.intValue(); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDGLengthStr = tsRawData.substring(124, 128); - BigInteger UDGLengthBigInt = new BigInteger(UDGLengthStr, 16); - ctx.ts.UDGLength = UDGLengthBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDGLengthStr = tsRawData.substring(164, 168); - BigInteger UDGLengthBigInt = new BigInteger(UDGLengthStr, 16); - ctx.ts.UDGLength = UDGLengthBigInt.intValue(); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDGChecksumStr = tsRawData.substring(128, 132); - BigInteger UDGChecksumBigInt = new BigInteger(UDGChecksumStr, 16); - ctx.ts.UDGChecksum = UDGChecksumBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDGChecksumStr = tsRawData.substring(168, 172); - BigInteger UDGChecksumBigInt = new BigInteger(UDGChecksumStr, 16); - ctx.ts.UDGChecksum = UDGChecksumBigInt.intValue(); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.InBandNetworkTelemetryData = tsRawData.substring(132, 196); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.InBandNetworkTelemetryData = tsRawData.substring(172, 236); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDPIntShimHeaderData = tsRawData.substring(132, 140); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDPIntShimHeaderData = tsRawData.substring(172, 180); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDPIntShimHeaderType = tsRawData.substring(132, 133); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDPIntShimHeaderType = tsRawData.substring(172, 173); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String udpIntShimNPTStr = tsRawData.substring(133, 134); - BigInteger udpIntShimNPTBigInt = new BigInteger(udpIntShimNPTStr, 16); - BigInteger bitwiseAnd = new BigInteger('C', 16); - BigInteger result = udpIntShimNPTBigInt.and(bitwiseAnd); - ctx.ts.UDPIntShimHeaderNPT = result.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String udpIntShimNPTStr = tsRawData.substring(173, 174); - BigInteger udpIntShimNPTBigInt = new BigInteger(udpIntShimNPTStr, 16); - BigInteger bitwiseAnd = new BigInteger('C', 16); - BigInteger result = udpIntShimNPTBigInt.and(bitwiseAnd); - ctx.ts.UDPIntShimHeaderNPT = result.intValue(); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDPIntShimHeaderRes1 = tsRawData.substring(133, 134); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDPIntShimHeaderRes1 = tsRawData.substring(174, 175); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String lengthStr = tsRawData.substring(134, 136); - BigInteger lengthStrBigInt = new BigInteger(lengthStr, 16); - ctx.ts.UDPIntShimHeaderLength= lengthStrBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String lengthStr = tsRawData.substring(174, 176); - BigInteger lengthStrBigInt = new BigInteger(lengthStr, 16); - ctx.ts.UDPIntShimHeaderLength = lengthStrBigInt.intValue(); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDPIntShimHeaderRes2 = tsRawData.substring(136, 138); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDPIntShimHeaderRes2 = tsRawData.substring(176, 178); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String nextProtoStr = tsRawData.substring(138, 140); - BigInteger nextProtoBigInt = new BigInteger(nextProtoStr, 16); - ctx.ts.UDPIntShimHeaderNextProto = nextProtoBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String nextProtoStr = tsRawData.substring(178, 180); - BigInteger nextProtoBigInt = new BigInteger(nextProtoStr, 16); - ctx.ts.UDPIntShimHeaderNextProto = nextProtoBigInt.intValue(); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.INTMetadataHeaderData = tsRawData.substring(140, 164); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.INTMetadataHeaderData = tsRawData.substring(180, 204); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderVersionStr = tsRawData.substring(140, 141); - BigInteger UDPIntMetaDataHeaderVersionStrBigInt = new BigInteger(UDPIntMetaDataHeaderVersionStr, 16); - ctx.ts.UDPIntMetaDataHeaderVersion = UDPIntMetaDataHeaderVersionStrBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderVersionStr = tsRawData.substring(180, 181); - BigInteger UDPIntMetaDataHeaderVersionStrBigInt = new BigInteger(UDPIntMetaDataHeaderVersionStr, 16); - ctx.ts.UDPIntMetaDataHeaderVersion = UDPIntMetaDataHeaderVersionStrBigInt.intValue(); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderRes = tsRawData.substring(141, 142); - BigInteger UDPIntMetaDataHeaderResBigInt = new BigInteger(UDPIntMetaDataHeaderRes, 16); - BigInteger bitwiseAnd = new BigInteger('C', 16); - BigInteger result = UDPIntMetaDataHeaderResBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderRes = result.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderRes = tsRawData.substring(181, 182); - BigInteger UDPIntMetaDataHeaderResBigInt = new BigInteger(UDPIntMetaDataHeaderRes, 16); - BigInteger bitwiseAnd = new BigInteger('C', 16); - BigInteger result = UDPIntMetaDataHeaderResBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderRes = result.intValue(); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderD = tsRawData.substring(141, 142); - BigInteger UDPIntMetaDataHeaderDBigInt = new BigInteger(UDPIntMetaDataHeaderD, 16); - BigInteger bitwiseAnd = new BigInteger('2', 16); - BigInteger result = UDPIntMetaDataHeaderDBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderD = result.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderD = tsRawData.substring(181, 182); - BigInteger UDPIntMetaDataHeaderDBigInt = new BigInteger(UDPIntMetaDataHeaderD, 16); - BigInteger bitwiseAnd = new BigInteger('2', 16); - BigInteger result = UDPIntMetaDataHeaderDBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderD = result.intValue(); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderE = tsRawData.substring(141, 142); - BigInteger UDPIntMetaDataHeaderEBigInt = new BigInteger(UDPIntMetaDataHeaderE, 16); - BigInteger bitwiseAnd = new BigInteger('1', 16); - BigInteger result = UDPIntMetaDataHeaderEBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderE = result.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderE = tsRawData.substring(181, 182); - BigInteger UDPIntMetaDataHeaderEBigInt = new BigInteger(UDPIntMetaDataHeaderE, 16); - BigInteger bitwiseAnd = new BigInteger('1', 16); - BigInteger result = UDPIntMetaDataHeaderEBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderE = result.intValue(); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderM = tsRawData.substring(142, 143); - BigInteger UDPIntMetaDataHeaderMBigInt = new BigInteger(UDPIntMetaDataHeaderM, 16); - BigInteger bitwiseAnd = new BigInteger('8', 16); - BigInteger result = UDPIntMetaDataHeaderMBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderM = result.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderM = tsRawData.substring(182, 183); - BigInteger UDPIntMetaDataHeaderMBigInt = new BigInteger(UDPIntMetaDataHeaderM, 16); - BigInteger bitwiseAnd = new BigInteger('8', 16); - BigInteger result = UDPIntMetaDataHeaderMBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderM = result.intValue(); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderPerhopMetadataLengthStr = tsRawData.substring(144, 146); - BigInteger UDPIntMetaDataHeaderPerhopMetadataLengthBigInt = new BigInteger(UDPIntMetaDataHeaderPerhopMetadataLengthStr, 16); - BigInteger bitwiseAnd = new BigInteger('1F', 16); - BigInteger result = UDPIntMetaDataHeaderPerhopMetadataLengthBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderPerhopMetadataLength = result.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderPerhopMetadataLengthStr = tsRawData.substring(184, 186); - BigInteger UDPIntMetaDataHeaderPerhopMetadataLengthBigInt = new BigInteger(UDPIntMetaDataHeaderPerhopMetadataLengthStr, 16); - BigInteger bitwiseAnd = new BigInteger('1F', 16); - BigInteger result = UDPIntMetaDataHeaderPerhopMetadataLengthBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderPerhopMetadataLength = result.intValue(); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderRemainingHopCntStr = tsRawData.substring(146, 148); - BigInteger UDPIntMetaDataHeaderRemainingHopCntBigInt = new BigInteger(UDPIntMetaDataHeaderRemainingHopCntStr, 16); - ctx.ts.UDPIntMetaDataHeaderRemainingHopCnt = UDPIntMetaDataHeaderRemainingHopCntBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderRemainingHopCntStr = tsRawData.substring(186, 188); - BigInteger UDPIntMetaDataHeaderRemainingHopCntBigInt = new BigInteger(UDPIntMetaDataHeaderRemainingHopCntStr, 16); - ctx.ts.UDPIntMetaDataHeaderRemainingHopCnt = UDPIntMetaDataHeaderRemainingHopCntBigInt.intValue(); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderInstructionBitmapStr = tsRawData.substring(148, 152); - BigInteger UDPIntMetaDataHeaderInstructionBitmapBigInt = new BigInteger(UDPIntMetaDataHeaderInstructionBitmapStr, 16); - ctx.ts.UDPIntMetaDataHeaderInstructionBitmap = UDPIntMetaDataHeaderInstructionBitmapBigInt.toString(16); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderInstructionBitmapStr = tsRawData.substring(188, 192); - BigInteger UDPIntMetaDataHeaderInstructionBitmapBigInt = new BigInteger(UDPIntMetaDataHeaderInstructionBitmapStr, 16); - ctx.ts.UDPIntMetaDataHeaderInstructionBitmap = UDPIntMetaDataHeaderInstructionBitmapBigInt.toString(16); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderDomainSpecificIDStr = tsRawData.substring(152, 156); - BigInteger UDPIntMetaDataHeaderDomainSpecificIDBigInt = new BigInteger(UDPIntMetaDataHeaderDomainSpecificIDStr, 16); - ctx.ts.UDPIntMetaDataHeaderDomainSpecificID = UDPIntMetaDataHeaderDomainSpecificIDBigInt.toString(16); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderDomainSpecificIDStr = tsRawData.substring(192, 196); - BigInteger UDPIntMetaDataHeaderDomainSpecificIDBigInt = new BigInteger(UDPIntMetaDataHeaderDomainSpecificIDStr, 16); - ctx.ts.UDPIntMetaDataHeaderDomainSpecificID = UDPIntMetaDataHeaderDomainSpecificIDBigInt.toString(16); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderDSInstructionStr = tsRawData.substring(156, 160); - BigInteger UDPIntMetaDataHeaderDSInstructionBigInt = new BigInteger(UDPIntMetaDataHeaderDSInstructionStr, 16); - ctx.ts.UDPIntMetaDataHeaderDSInstruction = UDPIntMetaDataHeaderDSInstructionBigInt.toString(16); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderDSInstructionStr = tsRawData.substring(196, 200); - BigInteger UDPIntMetaDataHeaderDSInstructionBigInt = new BigInteger(UDPIntMetaDataHeaderDSInstructionStr, 16); - ctx.ts.UDPIntMetaDataHeaderDSInstruction = UDPIntMetaDataHeaderDSInstructionBigInt.toString(16); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderDSFlagsStr = tsRawData.substring(160, 164); - BigInteger UDPIntMetaDataHeaderDSFlagsBigInt = new BigInteger(UDPIntMetaDataHeaderDSFlagsStr, 16); - ctx.ts.UDPIntMetaDataHeaderDSFlags = UDPIntMetaDataHeaderDSFlagsBigInt.toString(16); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderDSFlagsStr = tsRawData.substring(200, 204); - BigInteger UDPIntMetaDataHeaderDSFlagsBigInt = new BigInteger(UDPIntMetaDataHeaderDSFlagsStr, 16); - ctx.ts.UDPIntMetaDataHeaderDSFlags = UDPIntMetaDataHeaderDSFlagsBigInt.toString(16); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.INTMetadataStackData = tsRawData.substring(164, 196); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.INTMetadataStackData = tsRawData.substring(204, 236); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.INTMetadataStackSwitchID = tsRawData.substring(164, 172); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.INTMetadataStackSwitchID = tsRawData.substring(204, 212); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.INTMetadataSourceMetadata = tsRawData.substring(172, 196); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.INTMetadataSourceMetadata = tsRawData.substring(212, 236); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.INTMetadataSourceMetadataSwitchId = tsRawData.substring(172, 180); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.INTMetadataSourceMetadataSwitchId = tsRawData.substring(212, 220); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.INTMetadataSourceMetadataOriginatingMac = tsRawData.substring(180, 192); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.INTMetadataSourceMetadataOriginatingMac = tsRawData.substring(220, 232); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.INTMetadataSourceMetadataReserved = tsRawData.substring(192, 196); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.INTMetadataSourceMetadataReserved = tsRawData.substring(232, 236); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDP2Data = tsRawData.substring(196, 212); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDP2Data = tsRawData.substring(236, 252); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDP2SrcPortStr = tsRawData.substring(196, 200); - BigInteger UDP2SrcPortBigInt = new BigInteger(UDP2SrcPortStr,16); - ctx.ts.UDP2SrcPort = UDP2SrcPortBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDP2SrcPortStr = tsRawData.substring(236, 240); - BigInteger UDP2SrcPortBigInt = new BigInteger(UDP2SrcPortStr,16); - ctx.ts.UDP2SrcPort = UDP2SrcPortBigInt.intValue(); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDP2DstPortStr = tsRawData.substring(200, 204); - BigInteger UDP2DstPortBigInt = new BigInteger(UDP2DstPortStr,16); - ctx.ts.UDP2DstPort = UDP2DstPortBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDP2DstPortStr = tsRawData.substring(240, 244); - BigInteger UDP2DstPortBigInt = new BigInteger(UDP2DstPortStr,16); - ctx.ts.UDP2DstPort = UDP2DstPortBigInt.intValue(); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDP2Length = tsRawData.substring(204, 208); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDP2Length = tsRawData.substring(244, 248); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDP2Checksum = tsRawData.substring(208, 212); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDP2Checksum = tsRawData.substring(248, 252); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.dataII = tsRawData.substring(212, 252); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.dataII = tsRawData.substring(252, 292); - } - """ - } - } - ] -} diff --git a/snaps-hcp/OpenDistroElasticSearch/udp_data_parsing_pipeline.sh b/snaps-hcp/OpenDistroElasticSearch/udp_data_parsing_pipeline.sh deleted file mode 100755 index 24cc9652..00000000 --- a/snaps-hcp/OpenDistroElasticSearch/udp_data_parsing_pipeline.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash - -if [[ $# -ne 1 ]] ; then - echo "udp_data_parsing.sh : Missing ElasticSearch server address .... " - exit -fi - -curl -H 'Content-Type: application/json' -XPUT 'http://'$1'/_ingest/pipeline/ts_parsing' -d @udp_data_parsing.json diff --git a/snaps-hcp/Readme.md b/snaps-hcp/Readme.md deleted file mode 100644 index 5096c78b..00000000 --- a/snaps-hcp/Readme.md +++ /dev/null @@ -1,79 +0,0 @@ -# snaps-hcp Analytics Guide - -(deprecated) -The files contained within these directories are requried for setting up the -SNAPS Hortonworks analytics platform (aka. snap-hcp) which has been replaced -by a Siddhi engine and scripts (see docs/SIDDHI_AE_SETUP.md). - -1. Open source python program Espcap for live data capture. ( https://github.com/vichargrave/espcap ) -2. Open source distribution of ElasticSearch. ( https://opendistro.github.io/for-elasticsearch-docs/ ) - -## Requirements -**Espcap** relies on the Elasticsearch Python Client module to index packets in Elasticsearch. The version of the client module must match the version of Elasticsearch you want to use. -### Support for *Elasticsearch 7.x* requires: - -1. Python 3.7 (Python 2.7.x not supported) -2. TShark 3.0.1 (included in Wireshark) -3. *Click* module for Python -4. Elasticsearch Python Client module 7.x -5. Elasticsearch 7.x - -### Support for *Elasticsearch 6.x* requires: - -1. Requirements 1 - 3 listed above -2. Elasticsearch Python Client module 6.x -3. Elasticsearch 6.x - -## Installation -- The current setup in transparent security analytics tofino build environment is configured with an CENTOS AMI which has all the pre-requisites installed. -- To manually setup your CENTOS machine please follow the steps from [INSTALL](setup/INSTALL.md) Guide. - -## Running Examples - -- To start a live capture on an analytics instance from the network interface `ae-eth0`, get all packets and index them in the Elasticsearch cluster running at localhost:9200 , assuming your present working directory is *espcap* - ``` - cd espcap/ - sudo python3 src/espcap.py --nic=ae-eth0 --node=localhost:9200 --chunk=100 - ``` -The packets get captured and indexed in ElasticSearch under packets-* index. - -- Display the following help message: - ``` - cd espcap/ - espcap.py --help - Usage: espcap.py [OPTIONS] - - Options: - --node TEXT Elasticsearch IP and port (default=None, dump packets to - stdout) - --nic TEXT Network interface for live capture (default=None, if file - or dir specified) - --file TEXT PCAP file for file capture (default=None, if nic specified) - --dir TEXT PCAP directory for multiple file capture (default=None, if - nic specified) - --bpf TEXT Packet filter for live capture (default=all packets) - --chunk INTEGER Number of packets to bulk index (default=1000) - --count INTEGER Number of packets to capture during live capture - (default=0, capture indefinitely) - --list List the network interfaces - --help Show this message and exit. - ``` - -## Packet Indexing -Default installation of ElasticSearch cluster is at port 9200 and default installation of Kibana runs at port 5601. - -To visualize the indexed packets / incoming data in Kibana you need to create an Index pattern for incoming packets. This is a manual step. Do the following steps for the same. - - 1. To access Kibana UI go localhost:5601 => Stack Management => Index Patters => Create index pattern. - 2. Add a name for Index pattern and select the corresponding matching index source. - 3. Click on next step and choose timestamp as primary time field for the index pattern. - 4. Click create index pattern and navigate to Discover tab to see the incoming packets matching the index pattern. - - - - - - - - - diff --git a/snaps-hcp/setup/INSTALL.md b/snaps-hcp/setup/INSTALL.md deleted file mode 100644 index 6ad6af7b..00000000 --- a/snaps-hcp/setup/INSTALL.md +++ /dev/null @@ -1 +0,0 @@ -# TS Analytics environment Setup Guide for CentOS diff --git a/tests/trans_sec/analytics/__init__.py b/tests/trans_sec/analytics/__init__.py deleted file mode 100644 index e9b9345f..00000000 --- a/tests/trans_sec/analytics/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Unit tests for convert.py diff --git a/tests/trans_sec/analytics/oinc_tests.py b/tests/trans_sec/analytics/oinc_tests.py deleted file mode 100644 index 0269450f..00000000 --- a/tests/trans_sec/analytics/oinc_tests.py +++ /dev/null @@ -1,730 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Unit tests for http_session.py -import logging -import sys -import time -import unittest -from random import randrange, randint - -import ipaddress -import mock -from scapy.layers.inet import IP, UDP, TCP -from scapy.layers.inet6 import IPv6 -from scapy.layers.l2 import Ether - -from trans_sec import consts -from trans_sec.analytics import oinc -from trans_sec.analytics.oinc import SimpleAE -from trans_sec.packet.inspect_layer import ( - IntShim, IntMeta2, IntHeader, SourceIntMeta, IntMeta1, UdpInt, - TelemetryReport, DropReport) -from trans_sec.utils import tps_utils -from trans_sec.utils.http_session import HttpSession - -logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) - -logger = logging.getLogger('oinc_tests') - - -class SimpleAETests(unittest.TestCase): - """ - Unit tests for the class SimpleAE - """ - - def setUp(self): - self.ae = SimpleAE(mock.Mock(HttpSession), packet_count=20, - sample_interval=2) - self.sport = randrange(1000, 8000) - self.dport = randrange(1000, 8000) - self.ae_ip = '192.168.1.2' - self.dst_ipv4 = '10.1.0.2' - self.src_ipv4 = '10.1.0.6' - self.dst_ipv6 = ipaddress.ip_address( - '0000:0000:0000:0000:0000:0001:0000:0001') - self.src_ipv4 = '10.2.0.1' - self.src_ipv6 = ipaddress.ip_address( - '0000:0000:0000:0000:0000:0002:0000:0001') - self.dst_mac = rand_mac() - self.src_mac = rand_mac() - # self.orig_mac = rand_mac() - self.orig_mac = '00:00:00:02:02:00' - logger.info('Test sport - [%s] dport - [%s]', self.sport, self.dport) - - self.int_pkt_ipv4_udp = ( - Ether(src='00:00:00:00:01:01', dst=self.dst_mac) / - IP(dst=self.dst_ipv4, src=self.src_ipv4, - proto=consts.UDP_PROTO) / - UdpInt(dport=consts.UDP_INT_DST_PORT) / - IntShim(length=9, next_proto=consts.UDP_PROTO) / - IntHeader(meta_len=1) / - IntMeta1(switch_id=3) / - IntMeta2(switch_id=2) / - SourceIntMeta(switch_id=1, orig_mac=self.orig_mac) / - UDP(dport=self.dport, sport=self.sport) / - 'hello transparent-security' - ) - - self.ipv4_hash = tps_utils.create_attack_hash( - mac=self.orig_mac, port=self.dport, ip_addr=self.dst_ipv4, - ipv6_addr='::') - ip_len = (consts.IPV4_HDR_LEN + consts.UDP_INT_HDR_LEN - + consts.DRPT_LEN + consts.UDP_HDR_LEN - + consts.DRPT_PAYLOAD_LEN) - udp_int_len = ip_len - consts.IPV4_HDR_LEN - - self.int_drop_rpt_ipv4_udp = ( - Ether(type=consts.IPV4_TYPE) / - IP(dst=self.ae_ip, src=self.src_ipv4, len=ip_len, - proto=consts.UDP_PROTO) / - UdpInt(sport=consts.UDP_INT_SRC_PORT, - dport=consts.UDP_TRPT_DST_PORT, - len=udp_int_len) / - DropReport( - ver=consts.DRPT_VER, - node_id=0, - in_type=consts.DRPT_IN_TYPE, - rpt_len=consts.DRPT_REP_LEN, md_len=consts.DRPT_MD_LEN, - rep_md_bits=consts.DRPT_MD_BITS, - domain_id=consts.TRPT_DOMAIN_ID, - var_opt_bsmd=consts.DRPT_BS_MD, - timestamp=int(time.time()), - drop_count=5, - drop_hash=self.ipv4_hash) - ) - - self.int_pkt_ipv4_tcp = ( - Ether(src='00:00:00:00:01:01', dst=self.dst_mac) / - IP(dst=self.dst_ipv4, src=self.src_ipv4, - proto=consts.UDP_PROTO) / - UdpInt(dport=consts.UDP_INT_DST_PORT) / - IntShim(length=9, next_proto=consts.TCP_PROTO) / - IntHeader(meta_len=1) / - IntMeta1(switch_id=3) / - IntMeta2(switch_id=2) / - SourceIntMeta(switch_id=1, orig_mac=self.orig_mac) / - TCP(dport=self.dport, sport=self.sport) / - 'hello transparent-security' - ) - - self.int_pkt_ipv6_udp = ( - Ether(src='00:00:00:00:01:01', dst=self.dst_mac, - type=consts.IPV6_TYPE) / - IPv6(dst=self.dst_ipv6, - src=self.src_ipv6, - nh=consts.UDP_PROTO) / - UDP(dport=consts.UDP_INT_DST_PORT) / - IntShim(length=9, next_proto=consts.UDP_PROTO) / - IntHeader(meta_len=1) / - IntMeta1(switch_id=3) / - IntMeta2(switch_id=2) / - SourceIntMeta(switch_id=1, orig_mac=self.orig_mac) / - UDP(dport=self.dport, sport=self.sport) / - 'hello transparent-security' - ) - - self.int_pkt_ipv6_tcp = ( - Ether(src='00:00:00:00:01:01', dst=self.dst_mac, - type=consts.IPV6_TYPE) / - IPv6(dst=self.dst_ipv6, - src=self.src_ipv6, - nh=consts.UDP_PROTO) / - UDP(dport=consts.UDP_INT_DST_PORT) / - IntShim(length=9, next_proto=consts.TCP_PROTO) / - IntHeader(meta_len=1) / - IntMeta1(switch_id=3) / - IntMeta2(switch_id=2) / - SourceIntMeta(switch_id=1, orig_mac=self.orig_mac) / - TCP(dport=self.dport, sport=self.sport) / - 'hello transparent-security' - ) - - self.trpt_pkt_ipv4_out_ipv4_in_udp = ( - Ether(src=self.src_mac, dst=self.dst_mac) / - IP(dst=self.dst_ipv4, src=self.src_ipv4, - proto=consts.UDP_PROTO) / - UDP(sport=0, dport=consts.UDP_TRPT_DST_PORT, - # udp + telemetry header size - len=len(self.int_pkt_ipv4_udp) + 20 + 20) / - TelemetryReport(domain_id=consts.TRPT_DOMAIN_ID) / - self.int_pkt_ipv4_udp - ) - - self.trpt_pkt_ipv4_out_ipv6_in_udp = ( - Ether(src=self.src_mac, dst=self.dst_mac) / - IP(dst=self.dst_ipv4, src=self.src_ipv4, - proto=consts.UDP_PROTO) / - UDP(sport=0, dport=consts.UDP_TRPT_DST_PORT, - # udp + telemetry header size - len=len(self.int_pkt_ipv4_udp) + 20 + 20) / - TelemetryReport(domain_id=consts.TRPT_DOMAIN_ID) / - self.int_pkt_ipv6_udp - ) - - self.trpt_pkt_ipv4_out_ipv4_in_tcp = ( - Ether(src=self.src_mac, dst=self.dst_mac) / - IP(dst=self.dst_ipv4, src=self.src_ipv4, - proto=consts.UDP_PROTO) / - UDP(sport=0, dport=consts.UDP_TRPT_DST_PORT, - # udp + telemetry header size - len=len(self.int_pkt_ipv4_udp) + 20 + 20) / - TelemetryReport(domain_id=consts.TRPT_DOMAIN_ID) / - self.int_pkt_ipv4_tcp - ) - - self.trpt_pkt_ipv4_out_ipv6_in_tcp = ( - Ether(src=self.src_mac, dst=self.dst_mac) / - IP(dst=self.dst_ipv4, src=self.src_ipv4, - proto=consts.UDP_PROTO) / - UDP(sport=0, dport=consts.UDP_TRPT_DST_PORT, - # udp + telemetry header size - len=len(self.int_pkt_ipv4_udp) + 20 + 20) / - TelemetryReport(domain_id=consts.TRPT_DOMAIN_ID) / - self.int_pkt_ipv6_tcp - ) - - def test_extract_ipv4_udp_packet(self): - """ - Tests to ensure that an IPv4 UDP single packet will be parsed properly - """ - int_data = oinc.extract_int_data(self.int_pkt_ipv4_udp[Ether]) - self.assertEqual(self.orig_mac, int_data['devMac']) - self.assertEqual(self.src_ipv4, int_data['devAddr']) - self.assertEqual(self.dst_ipv4, int_data['dstAddr']) - self.assertEqual(self.dport, int_data['dstPort']) - self.assertEqual(consts.UDP_PROTO, int_data['protocol']) - - def test_extract_ipv4_udp_packet_trpt(self): - """ - Tests to ensure that an IPv4 UDP single packet will be parsed properly - """ - int_data = oinc.extract_trpt_data( - self.trpt_pkt_ipv4_out_ipv4_in_udp[UDP]) - self.assertEqual(self.orig_mac, int_data['devMac']) - self.assertEqual(self.src_ipv4, int_data['devAddr']) - self.assertEqual(self.dst_ipv4, int_data['dstAddr']) - self.assertEqual(self.dport, int_data['dstPort']) - self.assertEqual(consts.UDP_PROTO, int_data['protocol']) - - def test_extract_ipv4_udp_packet_drpt(self): - """ - Tests to ensure that an IPv4 UDP drop report will be parsed properly - """ - hash_key, count = oinc.extract_drop_rpt( - self.int_drop_rpt_ipv4_udp[UdpInt]) - self.assertEqual(self.ipv4_hash, hash_key) - self.assertEqual(5, count) - - def test_extract_ipv4_tcp_packet(self): - """ - Tests to ensure that an IPv4 UDP single packet will be parsed properly - """ - int_data = oinc.extract_int_data(self.int_pkt_ipv4_tcp[Ether]) - self.assertEqual(self.orig_mac, int_data['devMac']) - self.assertEqual(self.src_ipv4, int_data['devAddr']) - self.assertEqual(self.dst_ipv4, int_data['dstAddr']) - self.assertEqual(self.dport, int_data['dstPort']) - self.assertEqual(consts.TCP_PROTO, int_data['protocol']) - - def test_extract_ipv4_tcp_packet_trpt(self): - """ - Tests to ensure that an IPv4 UDP single packet will be parsed properly - """ - logger.debug('Packet to test - [%s]', - self.trpt_pkt_ipv4_out_ipv4_in_tcp) - int_data = oinc.extract_trpt_data( - self.trpt_pkt_ipv4_out_ipv4_in_tcp[UDP]) - self.assertEqual(self.orig_mac, int_data['devMac']) - self.assertEqual(self.src_ipv4, int_data['devAddr']) - self.assertEqual(self.dst_ipv4, int_data['dstAddr']) - self.assertEqual(self.dport, int_data['dstPort']) - self.assertEqual(consts.TCP_PROTO, int_data['protocol']) - - def test_extract_ipv6_udp_packet(self): - """ - Tests to ensure that an IPv4 UDP single packet will be parsed properly - """ - int_data = oinc.extract_int_data(self.int_pkt_ipv6_udp[Ether]) - self.assertEqual(self.orig_mac, int_data['devMac']) - self.assertEqual(str(self.src_ipv6), int_data['devAddr']) - self.assertEqual(str(self.dst_ipv6), int_data['dstAddr']) - self.assertEqual(self.dport, int_data['dstPort']) - self.assertEqual(consts.UDP_PROTO, int_data['protocol']) - - def test_extract_ipv6_udp_packet_trpt(self): - """ - Tests to ensure that an IPv6 UDP single packet will be parsed properly - """ - int_data = oinc.extract_trpt_data( - self.trpt_pkt_ipv4_out_ipv6_in_udp[UDP]) - self.assertEqual(self.orig_mac, int_data['devMac']) - self.assertEqual(str(self.src_ipv6), int_data['devAddr']) - self.assertEqual(str(self.dst_ipv6), int_data['dstAddr']) - self.assertEqual(self.dport, int_data['dstPort']) - self.assertEqual(consts.UDP_PROTO, int_data['protocol']) - - def test_extract_ipv6_tcp_packet(self): - """ - Tests to ensure that an IPv6 TCP single packet will be parsed properly - """ - int_data = oinc.extract_int_data(self.int_pkt_ipv6_tcp[Ether]) - self.assertEqual(self.orig_mac, int_data['devMac']) - self.assertEqual(str(self.src_ipv6), int_data['devAddr']) - self.assertEqual(str(self.dst_ipv6), int_data['dstAddr']) - self.assertEqual(self.dport, int_data['dstPort']) - self.assertEqual(consts.TCP_PROTO, int_data['protocol']) - - def test_extract_ipv6_tcp_packet_trpt(self): - """ - Tests to ensure that an IPv6 TCP single packet will be parsed properly - """ - int_data = oinc.extract_trpt_data( - self.trpt_pkt_ipv4_out_ipv6_in_tcp[UDP]) - self.assertEqual(self.orig_mac, int_data['devMac']) - self.assertEqual(str(self.src_ipv6), int_data['devAddr']) - self.assertEqual(str(self.dst_ipv6), int_data['dstAddr']) - self.assertEqual(self.dport, int_data['dstPort']) - self.assertEqual(consts.TCP_PROTO, int_data['protocol']) - - def test_process_single_drop_rpt_packet(self): - """ - Tests to ensure that an IPv4 UDP single packet is handled without Error - note: only testing via the handle_packet() API which would be called by - by the scapy sniffer thread - :return: - """ - self.assertFalse(self.ae.process_drop_rpt(self.int_pkt_ipv4_udp)) - - def test_process_single_ipv4_udp_packet(self): - """ - Tests to ensure that an IPv4 UDP single packet is handled without Error - note: only testing via the handle_packet() API which would be called by - by the scapy sniffer thread - :return: - """ - self.assertFalse(self.ae.process_packet(self.int_pkt_ipv4_udp)) - - def test_process_single_ipv4_udp_packet_trpt(self): - """ - Tests to ensure that an IPv4 UDP single packet is handled without Error - note: only testing via the handle_packet() API which would be called by - by the scapy sniffer thread - :return: - """ - self.assertFalse( - self.ae.process_packet(self.trpt_pkt_ipv4_out_ipv4_in_udp)) - - def test_process_single_ipv6_udp_packet(self): - """ - Tests to ensure that an IPv6 UDP single packet is handled without Error - note: only testing via the handle_packet() API which would be called by - by the scapy sniffer thread - :return: - """ - self.ae.process_packet(self.int_pkt_ipv6_udp) - - def test_process_single_ipv6_udp_packet_trpt(self): - """ - Tests to ensure that an IPv6 UDP single packet is handled without Error - note: only testing via the handle_packet() API which would be called by - by the scapy sniffer thread - :return: - """ - self.ae.process_packet( - self.trpt_pkt_ipv4_out_ipv6_in_udp, consts.UDP_TRPT_DST_PORT) - - def test_process_single_ipv4_tcp_packet(self): - """ - Tests to ensure that a single IPv4 TCP packet is handled without Error - note: only testing via the handle_packet() API which would be called by - by the scapy sniffer thread - :return: - """ - self.ae.process_packet(self.int_pkt_ipv4_tcp) - - def test_process_single_ipv4_tcp_packet_trpt(self): - """ - Tests to ensure that a single IPv4 TCP packet is handled without Error - note: only testing via the handle_packet() API which would be called by - by the scapy sniffer thread - :return: - """ - self.ae.process_packet( - self.trpt_pkt_ipv4_out_ipv4_in_tcp, consts.UDP_TRPT_DST_PORT) - - def test_process_single_ipv6_tcp_packet(self): - """ - Tests to ensure that a single IPv6 TCP packet is handled without Error - note: only testing via the handle_packet() API which would be called by - by the scapy sniffer thread - :return: - """ - self.ae.process_packet(self.int_pkt_ipv6_tcp) - - def test_process_single_ipv6_tcp_packet_trpt(self): - """ - Tests to ensure that a single IPv6 TCP packet is handled without Error - note: only testing via the handle_packet() API which would be called by - by the scapy sniffer thread - :return: - """ - self.ae.process_packet( - self.trpt_pkt_ipv4_out_ipv6_in_tcp, consts.UDP_TRPT_DST_PORT) - - def test_start_one_ipv4_udp_attack(self): - """ - Tests to ensure that one IPv4 UDP attack has been triggered - :return: - """ - for index in range(0, self.ae.packet_count + 1): - logger.debug('Processing packet #%s', index) - ret_val = self.ae.process_packet(self.int_pkt_ipv4_udp) - if index < self.ae.packet_count: - self.assertFalse(ret_val) - else: - self.assertTrue(ret_val) - - def test_start_one_ipv4_udp_attack_trpt(self): - """ - Tests to ensure that one IPv4 UDP attack has been triggered - :return: - """ - for index in range(0, self.ae.packet_count + 1): - logger.debug('Processing packet #%s', index) - ret_val = self.ae.process_packet( - self.trpt_pkt_ipv4_out_ipv4_in_udp, consts.UDP_TRPT_DST_PORT) - if index < self.ae.packet_count: - self.assertFalse(ret_val) - else: - self.assertTrue(ret_val) - - def test_ipv4_udp_proc_attack_and_drop(self): - """ - Tests to ensure that one IPv4 UDP attack has been triggered and has - released after processing several drop reports - :return: - """ - ret_val = False - for i in range(0, self.ae.packet_count + 1): - logger.debug('Processing packet #%s', i) - ret_val = self.ae.process_packet( - self.trpt_pkt_ipv4_out_ipv4_in_udp, consts.UDP_TRPT_DST_PORT) - if i < self.ae.packet_count: - self.assertFalse(ret_val) - else: - self.assertTrue(ret_val) - - # Ensure didn't loop again with False - self.assertTrue(ret_val) - - drop_ret = False - for j in range(0, 4): - drop_ret = self.ae.process_drop_rpt(self.int_drop_rpt_ipv4_udp) - if j < 3: - self.assertFalse(drop_ret) - else: - self.assertTrue(drop_ret) - - # Ensure didn't loop again with False - self.assertTrue(drop_ret) - - proc_ret = self.ae.process_packet( - self.trpt_pkt_ipv4_out_ipv4_in_udp, consts.UDP_TRPT_DST_PORT) - self.assertFalse(proc_ret) - - def test_start_one_ipv6_udp_attack(self): - """ - Tests to ensure that one IPv6 UDP attack has been triggered - :return: - """ - ret_val = False - for index in range(0, self.ae.packet_count + 1): - logger.debug('Processing packet #%s', index) - ret_val = self.ae.process_packet(self.int_pkt_ipv6_udp) - if index < self.ae.packet_count: - self.assertFalse(ret_val) - else: - self.assertTrue(ret_val) - self.assertTrue(ret_val) - - def test_start_one_ipv6_udp_attack_trpt(self): - """ - Tests to ensure that one IPv6 UDP attack has been triggered - :return: - """ - ret_val = False - for index in range(0, self.ae.packet_count + 1): - logger.debug('Processing packet #%s', index) - ret_val = self.ae.process_packet( - self.trpt_pkt_ipv4_out_ipv6_in_udp, consts.UDP_TRPT_DST_PORT) - if index < self.ae.packet_count: - self.assertFalse(ret_val) - else: - self.assertTrue(ret_val) - self.assertTrue(ret_val) - - def test_start_one_ipv4_tcp_attack(self): - """ - Tests to ensure that one IPv4 TCP attack has been triggered - :return: - """ - for index in range(0, self.ae.packet_count + 1): - logger.debug('Processing packet #%s', index) - ret_val = self.ae.process_packet(self.int_pkt_ipv4_tcp) - if index < self.ae.packet_count: - self.assertFalse(ret_val) - else: - self.assertTrue(ret_val) - - def test_start_one_ipv4_tcp_attack_trpt(self): - """ - Tests to ensure that one IPv4 TCP attack has been triggered - :return: - """ - ret_val = False - for index in range(0, self.ae.packet_count + 1): - logger.debug('Processing packet #%s', index) - ret_val = self.ae.process_packet( - self.trpt_pkt_ipv4_out_ipv4_in_tcp, consts.UDP_TRPT_DST_PORT) - if index < self.ae.packet_count: - self.assertFalse(ret_val) - else: - self.assertTrue(ret_val) - - self.assertTrue(ret_val) - - def test_start_one_ipv6_tcp_attack(self): - """ - Tests to ensure that one IPv6 TCP attack has been triggered - :return: - """ - ret_val = False - for index in range(0, self.ae.packet_count + 1): - logger.debug('Processing packet #%s', index) - ret_val = self.ae.process_packet(self.int_pkt_ipv6_tcp) - if index < self.ae.packet_count: - self.assertFalse(ret_val) - else: - self.assertTrue(ret_val) - self.assertTrue(ret_val) - - def test_start_one_ipv6_tcp_attack_trpt(self): - """ - Tests to ensure that one IPv6 TCP attack has been triggered - :return: - """ - ret_val = False - for index in range(0, self.ae.packet_count + 1): - logger.debug('Processing packet #%s', index) - ret_val = self.ae.process_packet( - self.trpt_pkt_ipv4_out_ipv6_in_tcp, consts.UDP_TRPT_DST_PORT) - if index < self.ae.packet_count: - self.assertFalse(ret_val) - else: - self.assertTrue(ret_val) - self.assertTrue(ret_val) - - def test_start_two_ipv4_udp_attacks(self): - """ - Tests to ensure that two IPv4 UDP attacks have been triggered - :return: - """ - pkt1 = (Ether(src='00:00:00:00:01:01', dst=self.dst_mac) / - IP(dst=self.dst_ipv4, src=self.src_ipv4, - proto=consts.UDP_PROTO) / - UdpInt() / - IntShim(length=9, next_proto=consts.UDP_PROTO) / - IntHeader(meta_len=1) / - IntMeta1(switch_id=3) / - IntMeta2(switch_id=2) / - SourceIntMeta(switch_id=1, orig_mac=self.orig_mac) / - UDP(dport=self.dport, sport=self.sport) / - 'hello transparent-security') - - pkt2 = (Ether(src='00:00:00:00:01:01', dst=self.dst_mac) / - IP(dst=self.dst_ipv4, src=self.src_ipv4, - proto=consts.UDP_PROTO) / - UdpInt() / - IntShim(length=9, next_proto=consts.UDP_PROTO) / - IntHeader(meta_len=1) / - IntMeta1(switch_id=3) / - IntMeta2(switch_id=2) / - SourceIntMeta(switch_id=1, orig_mac=self.orig_mac) / - UDP(dport=self.dport, sport=self.sport) / - 'hello transparent-security') - - for index in range(0, self.ae.packet_count): - logger.info('Iteration #%s', index) - ret_val1 = self.ae.process_packet(pkt1) - ret_val2 = self.ae.process_packet(pkt2) - logger.info('Checking index - [%s] - count - [%s]', - index, self.ae.packet_count) - if index * 2 < self.ae.packet_count: - logger.info('Expecting false - [%s]', ret_val1) - self.assertFalse(ret_val1) - self.assertFalse(ret_val2) - else: - logger.info('Expecting true - [%s]', ret_val1) - self.assertTrue(ret_val1) - self.assertTrue(ret_val2) - - def test_start_two_ipv6_udp_attacks(self): - """ - Tests to ensure that two IPv6 UDP attacks have been triggered - :return: - """ - pkt1 = (Ether(src='00:00:00:00:01:01', dst=self.dst_mac, - type=consts.IPV6_TYPE) / - IPv6(dst=self.dst_ipv6, - src=self.src_ipv6, - nh=consts.UDP_PROTO) / - UdpInt() / - IntShim(length=9, next_proto=consts.UDP_PROTO) / - IntHeader(meta_len=1) / - IntMeta1(switch_id=3) / - IntMeta2(switch_id=2) / - SourceIntMeta(switch_id=1, orig_mac=self.orig_mac) / - UDP(dport=self.dport, sport=self.sport) / - 'hello transparent-security') - - pkt2 = (Ether(src='00:00:00:00:01:01', dst=self.dst_mac, - type=consts.IPV6_TYPE) / - IPv6(dst=self.dst_ipv6, - src=self.src_ipv6, - nh=consts.UDP_PROTO) / - UdpInt() / - IntShim(length=9, next_proto=consts.UDP_PROTO) / - IntHeader(meta_len=1) / - IntMeta1(switch_id=3) / - IntMeta2(switch_id=2) / - SourceIntMeta(switch_id=1, orig_mac=self.orig_mac) / - UDP(dport=self.dport, sport=self.sport) / - 'hello transparent-security') - - for index in range(0, self.ae.packet_count): - logger.info('Iteration #%s', index) - ret_val1 = self.ae.process_packet(pkt1) - ret_val2 = self.ae.process_packet(pkt2) - logger.info('Checking index - [%s] - count - [%s]', - index, self.ae.packet_count) - if index * 2 < self.ae.packet_count: - logger.info('Expecting false - [%s]', ret_val1) - self.assertFalse(ret_val1) - self.assertFalse(ret_val2) - else: - logger.info('Expecting true - [%s]', ret_val1) - self.assertTrue(ret_val1) - self.assertTrue(ret_val2) - - def test_start_two_ipv4_tcp_attacks(self): - """ - Tests to ensure that two IPv4 UDP attacks have been triggered - :return: - """ - pkt1 = (Ether(src='00:00:00:00:01:01', dst=self.dst_mac) / - IP(dst=self.dst_ipv4, src=self.src_ipv4, - proto=consts.UDP_PROTO) / - UdpInt() / - IntShim(length=9, next_proto=consts.TCP_PROTO) / - IntHeader(meta_len=1) / - IntMeta1(switch_id=3) / - IntMeta2(switch_id=2) / - SourceIntMeta(switch_id=1, orig_mac=self.orig_mac) / - TCP(dport=self.dport, sport=self.sport) / - 'hello transparent-security') - - pkt2 = (Ether(src='00:00:00:00:01:01', dst=self.dst_mac) / - IP(dst=self.dst_ipv4, src=self.src_ipv4, - proto=consts.UDP_PROTO) / - UdpInt() / - IntShim(length=9, next_proto=consts.TCP_PROTO) / - IntHeader(meta_len=1) / - IntMeta1(switch_id=3) / - IntMeta2(switch_id=2) / - SourceIntMeta(switch_id=1, orig_mac=self.orig_mac) / - TCP(dport=self.dport, sport=self.sport) / - 'hello transparent-security') - - for index in range(0, self.ae.packet_count): - logger.info('Iteration #%s', index) - ret_val1 = self.ae.process_packet(pkt1) - ret_val2 = self.ae.process_packet(pkt2) - logger.info('Checking index - [%s] - count - [%s]', - index, self.ae.packet_count) - if index * 2 < self.ae.packet_count: - logger.info('Expecting false - [%s]', ret_val1) - self.assertFalse(ret_val1) - self.assertFalse(ret_val2) - else: - logger.info('Expecting true - [%s]', ret_val1) - self.assertTrue(ret_val1) - self.assertTrue(ret_val2) - - def test_start_two_ipv6_tcp_attacks(self): - """ - Tests to ensure that two IPv6 UDP attacks have been triggered - :return: - """ - pkt1 = (Ether(src='00:00:00:00:01:01', dst=self.dst_mac, - type=consts.IPV6_TYPE) / - IPv6(dst=self.dst_ipv6, - src=self.src_ipv6, - nh=consts.UDP_PROTO) / - UdpInt() / - IntShim(length=9, next_proto=consts.TCP_PROTO) / - IntHeader(meta_len=1) / - IntMeta1(switch_id=3) / - IntMeta2(switch_id=2) / - SourceIntMeta(switch_id=1, orig_mac=self.orig_mac) / - TCP(dport=self.dport, sport=self.sport) / - 'hello transparent-security') - - pkt2 = (Ether(src='00:00:00:00:01:01', dst=self.dst_mac, - type=consts.IPV6_TYPE) / - IPv6(dst=self.dst_ipv6, - src=self.src_ipv6, - nh=consts.UDP_PROTO) / - UdpInt() / - IntShim(length=9, next_proto=consts.TCP_PROTO) / - IntHeader(meta_len=1) / - IntMeta1(switch_id=3) / - IntMeta2(switch_id=2) / - SourceIntMeta(switch_id=1, orig_mac=self.orig_mac) / - TCP(dport=self.dport, sport=self.sport) / - 'hello transparent-security') - - for index in range(0, self.ae.packet_count): - logger.info('Iteration #%s', index) - ret_val1 = self.ae.process_packet(pkt1) - ret_val2 = self.ae.process_packet(pkt2) - logger.info('Checking index - [%s] - count - [%s]', - index, self.ae.packet_count) - if index * 2 < self.ae.packet_count: - logger.info('Expecting false - [%s]', ret_val1) - self.assertFalse(ret_val1) - self.assertFalse(ret_val2) - else: - logger.info('Expecting true - [%s]', ret_val1) - self.assertTrue(ret_val1) - self.assertTrue(ret_val2) - - -def rand_mac(): - return "%02x:%02x:%02x:%02x:%02x:%02x" % ( - randint(0, 255), - randint(0, 255), - randint(0, 255), - randint(0, 255), - randint(0, 255), - randint(0, 255) - ) diff --git a/tests/trans_sec/controller/aggregate_controller_tests.py b/tests/trans_sec/controller/aggregate_controller_tests.py deleted file mode 100644 index f7323aa1..00000000 --- a/tests/trans_sec/controller/aggregate_controller_tests.py +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Unit tests for convert.py -import logging -import mock -import pkg_resources -import unittest - -import yaml - -from trans_sec.controller.aggregate_controller import AggregateController - -logger = logging.getLogger('aggregate_controller_tests') - - -class AggregateControllerTests(unittest.TestCase): - """ - Tests for the CoreController class - """ - def setUp(self): - # Parse topology file and store into object - topo_file = pkg_resources.resource_filename( - 'tests.trans_sec.conf', 'test_topology.yaml') - with open(topo_file, 'r') as f: - self.topo = yaml.load(f) - logger.info("Opened file - %s" % f.name) - - @mock.patch('trans_sec.p4runtime_lib.helper.P4InfoHelper', - return_value=mock.Mock()) - @mock.patch('trans_sec.p4runtime_lib.p4rt_switch.P4RuntimeSwitch', - return_value=mock.Mock()) - def test_construction(self, m1, m2): - """ - Tests constructor for class CoreController - """ - controller = AggregateController('bmv2', 'config_dir', self.topo, - '/tmp') - self.assertIsNotNone(controller) diff --git a/tests/trans_sec/controller/core_controller_tests.py b/tests/trans_sec/controller/core_controller_tests.py deleted file mode 100644 index 13913caa..00000000 --- a/tests/trans_sec/controller/core_controller_tests.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Unit tests for convert.py -import logging -import mock -import pkg_resources -import unittest - -import yaml - -from trans_sec.controller.core_controller import CoreController - -logger = logging.getLogger('core_controller_test') - - -class CoreControllerTests(unittest.TestCase): - """ - Tests for the CoreController class - """ - def setUp(self): - # Parse topology file and store into object - topo_file = pkg_resources.resource_filename( - 'tests.trans_sec.conf', 'test_topology.yaml') - with open(topo_file, 'r') as f: - self.topo = yaml.load(f) - logger.info("Opened file - %s" % f.name) - - @mock.patch('trans_sec.p4runtime_lib.helper.P4InfoHelper', - return_value=mock.Mock()) - @mock.patch('trans_sec.p4runtime_lib.p4rt_switch.P4RuntimeSwitch', - return_value=mock.Mock()) - def test_construction(self, m1, m2): - """ - Tests constructor for class CoreController - """ - controller = CoreController('bmv2', 'config_dir', self.topo, '/tmp') - self.assertIsNotNone(controller) diff --git a/tests/trans_sec/controller/ddos_sdn_controller_tests.py b/tests/trans_sec/controller/ddos_sdn_controller_tests.py deleted file mode 100644 index 395f6032..00000000 --- a/tests/trans_sec/controller/ddos_sdn_controller_tests.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Unit tests for http_session.py -import logging -import unittest - -import pkg_resources - -from trans_sec.controller.ddos_sdn_controller import DdosSdnController - -logger = logging.getLogger('ddos_sdn_controller_tests') - - -class HttpSessionTests(unittest.TestCase): - """ - Unit tests for utility functions in convert.py - """ - - # def setUp(self): - # """ - # Start HTTP server - # :return: - # """ - # logging.basicConfig(level=logging.DEBUG) - # topo_file = pkg_resources.resource_filename( - # 'tests.trans_sec.conf', 'test_topology.json') - # mock_switch_conf_dir = None - # self.controller = DdosSdnController( - # topo_file, mock_switch_conf_dir, 9998, 'scenario1', '/tmp') - # self.controller.start() - # - # def tearDown(self): - # self.controller.stop() - - # def test_foo(self): - # self.controller.add_attacker({}) diff --git a/tests/trans_sec/controller/gateway_controller_tests.py b/tests/trans_sec/controller/gateway_controller_tests.py deleted file mode 100644 index 40577df1..00000000 --- a/tests/trans_sec/controller/gateway_controller_tests.py +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Unit tests for convert.py -import logging -import mock -import pkg_resources -import unittest - -import yaml - -from trans_sec.controller.gateway_controller import GatewayController - -logger = logging.getLogger('gateway_controller_tests') - - -class GatewayControllerTests(unittest.TestCase): - """ - Tests for the CoreController class - """ - def setUp(self): - # Parse topology file and store into object - topo_file = pkg_resources.resource_filename( - 'tests.trans_sec.conf', 'test_topology.yaml') - with open(topo_file, 'r') as f: - self.topo = yaml.load(f) - logger.info("Opened file - %s" % f.name) - - @mock.patch('trans_sec.p4runtime_lib.helper.P4InfoHelper', - return_value=mock.Mock()) - @mock.patch('trans_sec.p4runtime_lib.p4rt_switch.P4RuntimeSwitch', - return_value=mock.Mock()) - def test_construction(self, m1, m2): - """ - Tests constructor for class CoreController - """ - controller = GatewayController('bmv2', 'config_dir', self.topo, '/tmp') - self.assertIsNotNone(controller) - controller.make_switch_rules(True) diff --git a/tests/trans_sec/mininet/__init__.py b/tests/trans_sec/mininet/__init__.py deleted file mode 100644 index e9b9345f..00000000 --- a/tests/trans_sec/mininet/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Unit tests for convert.py diff --git a/tests/trans_sec/mininet/exercise_tests.py b/tests/trans_sec/mininet/exercise_tests.py deleted file mode 100644 index 8cc2d8dd..00000000 --- a/tests/trans_sec/mininet/exercise_tests.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -import unittest - -logger = logging.getLogger('exercise_tests') - - -class ExerciseTopoTests(unittest.TestCase): - """ - Stub for testing the ExerciseTopo class - """ diff --git a/tests/trans_sec/p4runtime_lib/__init__.py b/tests/trans_sec/p4runtime_lib/__init__.py deleted file mode 100644 index 77c9835a..00000000 --- a/tests/trans_sec/p4runtime_lib/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/tests/trans_sec/p4runtime_lib/aggregate_switch_tests.py b/tests/trans_sec/p4runtime_lib/aggregate_switch_tests.py deleted file mode 100644 index 892711fb..00000000 --- a/tests/trans_sec/p4runtime_lib/aggregate_switch_tests.py +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import logging -import sys -import unittest -import mock -import pkg_resources -import yaml - -from trans_sec.p4runtime_lib.aggregate_switch import AggregateSwitch - -logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) -logger = logging.getLogger('aggregate_switch_tests') - - -class AggregateSwitchTests(unittest.TestCase): - """ - Tests for the CoreController class - """ - - def setUp(self): - topo_file = pkg_resources.resource_filename( - 'tests.trans_sec.conf', 'aggregate-topo.yaml') - with open(topo_file, 'r') as f: - topo_dict = yaml.safe_load(f) - - print(topo_dict) - self.sw_info = topo_dict['switches']['aggregate'] - self.p4_json = pkg_resources.resource_filename( - 'tests.trans_sec.conf', 'aggregate.json') - - @mock.patch('trans_sec.p4runtime_lib.helper.P4InfoHelper', - return_value=mock.Mock()) - @mock.patch( - 'trans_sec.switch.SwitchConnection.build_device_config', - return_value=mock.Mock()) - def test_construction(self, m1, m2): - """ - Tests constructor for class CoreController - """ - - switch = AggregateSwitch(sw_info=self.sw_info) - self.assertIsNotNone(switch) diff --git a/tests/trans_sec/p4runtime_lib/core_switch_tests.py b/tests/trans_sec/p4runtime_lib/core_switch_tests.py deleted file mode 100644 index dcb2588d..00000000 --- a/tests/trans_sec/p4runtime_lib/core_switch_tests.py +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import logging -import sys -import unittest -import mock -import pkg_resources -import yaml - -from trans_sec.p4runtime_lib.core_switch import CoreSwitch - -logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) -logger = logging.getLogger('core_switch_tests') - - -class CoreSwitchTests(unittest.TestCase): - """ - Tests for the CoreController class - """ - - def setUp(self): - topo_file = pkg_resources.resource_filename( - 'tests.trans_sec.conf', 'core-topo.yaml') - with open(topo_file, 'r') as f: - topo_dict = yaml.safe_load(f) - - print(topo_dict) - self.sw_info = topo_dict['switches']['core'] - self.p4_json = pkg_resources.resource_filename( - 'tests.trans_sec.conf', 'core.json') - - @mock.patch( - 'trans_sec.p4runtime_lib.helper.P4InfoHelper', - return_value=mock.Mock()) - @mock.patch( - 'trans_sec.switch.SwitchConnection.build_device_config', - return_value=mock.Mock()) - def test_construction(self, m1, m2): - """ - Tests constructor for class CoreController - """ - - switch = CoreSwitch(sw_info=self.sw_info) - self.assertIsNotNone(switch) diff --git a/tests/trans_sec/p4runtime_lib/gateway_switch_tests.py b/tests/trans_sec/p4runtime_lib/gateway_switch_tests.py deleted file mode 100644 index b3be31a3..00000000 --- a/tests/trans_sec/p4runtime_lib/gateway_switch_tests.py +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import logging -import sys -import unittest -import mock -import pkg_resources -import yaml - -from trans_sec.p4runtime_lib.gateway_switch import GatewaySwitch - -logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) -logger = logging.getLogger('gateway_switch_tests') - - -class GatewaySwitchTests(unittest.TestCase): - """ - Tests for the CoreController class - """ - - def setUp(self): - topo_file = pkg_resources.resource_filename( - 'tests.trans_sec.conf', 'gateway-topo.yaml') - with open(topo_file, 'r') as f: - topo_dict = yaml.safe_load(f) - - print(topo_dict) - self.sw_info = topo_dict['switches']['gateway'] - self.p4_json = pkg_resources.resource_filename( - 'tests.trans_sec.conf', 'gateway.json') - - @mock.patch('trans_sec.p4runtime_lib.helper.P4InfoHelper', - return_value=mock.Mock()) - @mock.patch( - 'trans_sec.switch.SwitchConnection.build_device_config', - return_value=mock.Mock()) - def test_construction(self, m1, m2): - """ - Tests constructor for class CoreController - """ - - switch = GatewaySwitch(sw_info=self.sw_info) - self.assertIsNotNone(switch) diff --git a/trans_sec/analytics/__init__.py b/trans_sec/analytics/__init__.py deleted file mode 100644 index e9b9345f..00000000 --- a/trans_sec/analytics/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Unit tests for convert.py diff --git a/trans_sec/analytics/oinc.py b/trans_sec/analytics/oinc.py deleted file mode 100644 index eb2767ec..00000000 --- a/trans_sec/analytics/oinc.py +++ /dev/null @@ -1,626 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import abc -import datetime -import ipaddress -import logging -import threading -import time - -from time import strftime, localtime -from anytree import search, Node, RenderTree -from scapy.all import sniff -from scapy.layers.inet import IP, UDP, TCP -from scapy.layers.inet6 import IPv6 -from scapy.layers.l2 import Ether - -from trans_sec.consts import (UDP_PROTO, UDP_TRPT_DST_PORT, IPV4_TYPE, - UDP_INT_DST_PORT, IPV6_TYPE) -from trans_sec.packet.inspect_layer import ( - IntHeader, IntMeta1, IntMeta2, IntShim, SourceIntMeta, TelemetryReport, - EthInt, DropReport) -from trans_sec.utils import tps_utils - -logger = logging.getLogger('oinc') - - -class PacketAnalytics(object): - """ - Analytics Engine class - """ - - def __init__(self, sdn_interface, packet_count=100, sample_interval=60, - sdn_attack_context='gwAttack'): - """ - Constructor - :param sdn_interface: the HTTP interface to the SDN Controller - :param packet_count: the number of packets to trigger an attack - :param sample_interval: the interval in seconds used for counting the - packets - """ - self.sdn_interface = sdn_interface - self.packet_count = packet_count - self.sample_interval = sample_interval - self.count_map = dict() - self.bytes_map = dict() - self.sniff_stop = threading.Event() - self.sdn_attack_context = sdn_attack_context - - logger.debug("Started AE with attack call to [%s/%s]", - self.sdn_interface, self.sdn_attack_context) - - def start_sniffing(self, iface, drop_iface=None, - udp_dport=UDP_TRPT_DST_PORT): - """ - Starts the sniffer thread - :param iface: the interface to sniff for telemetry reports - :param drop_iface: the interface to sniff for drop reports - :param udp_dport: the UDP dport to sniff (default 555) - """ - logger.info('Starting INT packet listener int_thread') - int_thread = threading.Thread(target=self.sniff_int, - args=(iface, udp_dport)) - int_thread.start() - logger.debug('int_thread has been started') - - if drop_iface: - logger.info('Starting Drop Report listener dr_thread') - dr_thread = threading.Thread(target=self.sniff_drop, - args=(drop_iface, 'foo')) - dr_thread.start() - logger.debug('dr_thread has been started') - - def sniff_int(self, iface, udp_port): - logger.info("INT monitoring iface [%s]", iface) - - try: - sniff(iface=iface, - prn=lambda packet: self.handle_packet(packet, udp_port), - stop_filter=lambda p: self.sniff_stop.is_set()) - except Exception as e: - logger.error('Unexpected error sniffing for INT Data - [%s]', e) - self.sniff_int(iface, udp_port) - - def sniff_drop(self, iface, foo=None): - logger.info("Drop monitoring iface [%s]", iface) - - try: - sniff(iface=iface, - prn=lambda packet: self.handle_drop_rpt(packet), - stop_filter=lambda p: self.sniff_stop.is_set()) - except Exception as e: - logger.error('Unexpected error sniffing for Drop Reports - [%s]', - e) - raise e - - def stop_sniffing(self): - """ - Stops the sniffer thread - """ - self.sniff_stop.set() - - def handle_packet(self, packet, udp_dport): - """ - Determines whether or not to process this packet - :param packet: the packet to process - :param udp_dport: the UDP protocol dport value to filter - :return T/F - True when an attack has been triggered - """ - return self.process_packet(packet, udp_dport) - - def handle_drop_rpt(self, packet): - """ - Determines whether or not to process this packet - :param packet: the packet to process - :return T/F - True when an attack has been triggered - """ - return self.process_drop_rpt(packet) - - def _send_attack(self, **attack_dict): - """ - Sends an HTTP POST to the SDN controllers HTTP interface 'attack' - :param attack_dict: the data to send - :raises Exception: due to the remote HTTP POST - """ - logger.info('Start attack - [%s] to [%s]', - attack_dict, self.sdn_attack_context) - self.sdn_interface.post(self.sdn_attack_context, attack_dict) - - def _stop_attack(self, **attack_dict): - """ - Sends an HTTP POST to the SDN controllers HTTP interface 'attack' - :param attack_dict: the data to send - :raises Exception: due to the remote HTTP POST - """ - logger.info('Stopping attack - [%s] to [%s]', - attack_dict, self.sdn_attack_context) - self.sdn_interface.delete(self.sdn_attack_context, attack_dict) - logger.info('Attack stopped at time [%s] with args - [%s]', - strftime("%b-%d-%Y %H:%M:%S", localtime()), attack_dict) - - @abc.abstractmethod - def process_packet(self, packet, udp_dport=UDP_INT_DST_PORT): - """ - Processes a packet to determine if an attack is occurring - :param packet: the packet to process - :param udp_dport: the UDP port value on which to filter - :return: T/F - True when an attack has been triggered - """ - return - - @abc.abstractmethod - def process_drop_rpt(self, packet): - """ - Processes a drop packet to determine if an attack no longer occurring - :param packet: the packet to process - :return: T/F - True when an attack should no longer be triggered - """ - return - - -def extract_int_data(ether_pkt): - """ - Parses the required data from the packet - :param ether_pkt: the packet to parse - :return: dict with choice header fields extracted - """ - logger.debug('Extracting packet - [%s]', ether_pkt.summary()) - if ether_pkt.type == IPV4_TYPE: - ip_pkt = IP(_pkt=ether_pkt.payload) - logger.debug('IPv4 dst - [%s], src - [%s], proto - [%s]', - ip_pkt.dst, ip_pkt.src, ip_pkt.proto) - elif ether_pkt.type == IPV6_TYPE: - ip_pkt = IPv6(_pkt=ether_pkt.payload) - logger.debug('IPv6 dst - [%s], src - [%s], nh - [%s]', - ip_pkt.dst, ip_pkt.src, ip_pkt.nh) - else: - logger.warning('Unable to process ether type - [%s]', ether_pkt.type) - return None - - udp_int_pkt = UDP(_pkt=ip_pkt.payload) - logger.debug('UDP INT sport - [%s], dport - [%s], len - [%s]', - udp_int_pkt.sport, udp_int_pkt.dport, udp_int_pkt.len) - int_shim_pkt = IntShim(_pkt=udp_int_pkt.payload) - logger.debug('INT Shim next_proto - [%s], npt - [%s], length - [%s]', - int_shim_pkt.next_proto, int_shim_pkt.npt, - int_shim_pkt.length) - int_hdr_pkt = IntHeader(_pkt=int_shim_pkt.payload) - - logger.debug('INT Header ver - [%s]', int_hdr_pkt.ver) - - if int_shim_pkt.length == 7: - source_int_pkt = SourceIntMeta(_pkt=int_hdr_pkt.payload) - elif int_shim_pkt.length == 8: - int_meta_1 = IntMeta1(_pkt=int_hdr_pkt.payload) - logger.debug('INT Meta 1 switch_id - [%s]', int_meta_1.switch_id) - source_int_pkt = SourceIntMeta(_pkt=int_meta_1.payload) - elif int_shim_pkt.length == 9: - int_meta_1 = IntMeta1(_pkt=int_hdr_pkt.payload) - logger.debug('INT Meta 1 switch_id - [%s]', int_meta_1.switch_id) - int_meta_2 = IntMeta2(_pkt=int_meta_1.payload) - logger.debug('INT Meta 2 switch_id - [%s]', int_meta_2.switch_id) - source_int_pkt = SourceIntMeta(_pkt=int_meta_2.payload) - else: - return - - logger.debug('SourceIntMeta switch_id - [%s], orig_mac - [%s]', - source_int_pkt.switch_id, source_int_pkt.orig_mac) - - if int_shim_pkt.next_proto == UDP_PROTO: - tcp_udp_pkt = UDP(_pkt=source_int_pkt.payload) - logger.debug('TCP sport - [%s], dport - [%s], len - [%s]', - tcp_udp_pkt.sport, tcp_udp_pkt.dport, tcp_udp_pkt.len) - else: - tcp_udp_pkt = TCP(_pkt=source_int_pkt.payload) - logger.debug('TCP sport - [%s], dport - [%s]', - tcp_udp_pkt.sport, tcp_udp_pkt.dport) - - orig_mac = source_int_pkt.orig_mac - - try: - out = dict( - devMac=orig_mac, - devAddr=ip_pkt.src, - dstAddr=ip_pkt.dst, - dstPort=tcp_udp_pkt.dport, - protocol=int_shim_pkt.next_proto, - packetLen=len(ether_pkt), - ) - except Exception as e: - logger.error('Error extracting header data - %s', e) - return None - logger.debug('Extracted header data [%s]', out) - return out - - -def extract_trpt_data(udp_packet): - """ - Parses the required data from the packet - :param udp_packet: the packet to parse - :return: dict with choice header fields extracted - """ - trpt_pkt = TelemetryReport(_pkt=udp_packet.payload) - trpt_eth = EthInt(trpt_pkt.payload) - logger.debug('TRPT ethernet dst - [%s], src - [%s], type - [%s]', - trpt_eth.dst, trpt_eth.src, trpt_eth.type) - return extract_int_data(trpt_eth) - - -def extract_drop_rpt(udp_packet): - """ - Parses the required data from the packet - :param udp_packet: the packet to parse - :return: tuple of key (hash|int) and value (count|int) - """ - logger.debug('UDP packet sport [%s], dport [%s], len [%s]', - udp_packet.sport, udp_packet.dport, udp_packet.len) - - drop_rpt = DropReport(_pkt=udp_packet.payload) - if drop_rpt: - return drop_rpt.drop_hash, drop_rpt.drop_count - - -class Oinc(PacketAnalytics): - """ - Oinc implementation of PacketAnalytics - """ - - def __init__(self, sdn_interface, packet_count=100, sample_interval=60): - super(self.__class__, self).__init__(sdn_interface, packet_count, - sample_interval) - self.tree = Node('root') - - def process_packet(self, packet, udp_dport=UDP_INT_DST_PORT): - mac, src_ip, dst_ip, dst_port, packet_size = self.__parse_tree(packet) - - if mac: - if src_ip and dst_ip and dst_port and packet_size: - self.__packet_with_mac(mac, src_ip, dst_ip, dst_port, - packet_size) - self.__manage_tree() - - def process_drop_rpt(self, packet): - pass - - def __parse_tree(self, packet): - """ - Processes a packet from a new device that has not been counted - """ - info = extract_int_data(packet[Ether]) - logger.info('Processing packet with info [%s]', info) - - macs = search.findall_by_attr(self.tree, info.get('srcMac'), - name='name', maxlevel=2, maxcount=1) - - mac = None - src_ip = None - dst_ip = None - dst_port = None - packet_size = None - - if len(macs) > 0: - mac = macs[0] - src_ips = search.findall_by_attr( - mac, info.get('srcIP'), name='name', maxlevel=2, maxcount=1) - if len(src_ips) != 0: - src_ip = src_ips[0] - dst_ips = search.findall_by_attr( - src_ip, info.get('dstIP'), name='name', maxlevel=2, - maxcount=1) - if len(dst_ips) != 0: - dst_ip = dst_ips[0] - logger.info('Processing source IPs - %s', src_ips) - dst_ports = search.findall_by_attr( - dst_ip, info.get('dstPort'), name='name', - maxlevel=2, maxcount=1) - if len(dst_ports) != 0: - dst_port = dst_ports[0] - packet_sizes = search.findall_by_attr( - dst_port, info.get('packet_size'), - name='name', maxlevel=2, maxcount=1) - if len(packet_sizes) != 0: - packet_size = packet_sizes[0] - - return mac, src_ip, dst_ip, dst_port, packet_size - - def __manage_tree(self): - """ - Updates the tree - I don't think this routine does anything at all - """ - for pre, fill, node in RenderTree(self.tree): - if node.name == 'count': - logger.info( - "Tree info %s%s: %s %s p/s attack: %s", - pre, node.name, node.value, node.pps, node.attack) - else: - logger.info("Pre - [%s], Fill - [%s], Node - [%s]", - pre, fill, node.name) - - def __packet_with_mac(self, mac, src_ip, dst_ip, dst_port, packet_size): - """ - Processes a packet from an existing device that has been counted - """ - logger.debug('Packet with MAC [%s] and source IP [%s]', mac, src_ip) - count = packet_size.children[0] - count.value = count.value + 1 - base_time = count.time - current_time = datetime.datetime.today() - delta = (current_time - base_time).total_seconds() - count.pps = count.value / delta - if (count.value > 3 and count.pps > 100 - and not count.attack): - logger.info('UDP Flood attack detected') - count.attack = True - - # Send to SDN - try: - self._send_attack(**dict( - src_mac=mac.name, - dst_ip=dst_ip.name, - dst_port=dst_port.name)) - except Exception as e: - logger.error('Unexpected error [%s]', e) - - if delta > 60: - count.time = current_time - count.value = 1 - - -class SimpleAE(PacketAnalytics): - """ - Simple implementation of PacketAnalytics where the count for detecting - attack notifications is based on the unique hash of the extracted INT data - """ - - def __init__(self, sdn_interface, packet_count=100, sample_interval=60, - sdn_attack_context='gwAttack', drop_count=3, - byte_count=50000): - super(self.__class__, self).__init__( - sdn_interface, packet_count, sample_interval, sdn_attack_context) - # Holds the last time an attack call was issued to the SDN controller - self.attack_map = dict() - self.attack_payload = dict() - - # same key as above, value tuple (int, int) first holds the drop count - # and the second holds the number of drop reports without receiving - # any associated packets - self.drop_rpt_map = dict() - self.drop_count = drop_count - self.byte_count = byte_count - - def process_packet(self, packet, udp_dport=UDP_INT_DST_PORT): - """ - Processes a packet to determine if an attack is occurring if the IP - protocol is as expected - :param packet: the packet to process - :param udp_dport: the UDP port value on which to filter - :return: T/F - True when an attack has been triggered - """ - ip_pkt, protocol, pkt_bytes = parse_ip_pkt(packet) - if ip_pkt and protocol and protocol == UDP_PROTO: - udp_packet = UDP(_pkt=ip_pkt.payload) - if udp_packet.dport == udp_dport and udp_dport == UDP_INT_DST_PORT: - int_data = extract_int_data(packet[Ether]) - if int_data: - return self.__process(int_data, pkt_bytes) - else: - logger.warning('Unable to debug INT data') - return False - elif (udp_packet.dport == udp_dport - and udp_dport == UDP_TRPT_DST_PORT): - int_data = extract_trpt_data(udp_packet) - if int_data: - return self.__process(int_data, pkt_bytes) - else: - logger.warning('Unable to debug INT data') - return False - else: - logger.debug( - 'Cannot process UDP packet dport of - [%s], expected - ' - '[%s]', udp_packet.dport, udp_dport) - return False - - def __process(self, int_data, pkt_bytes): - """ - Processes INT data for analysis - :param int_data: the data to process - :return: - """ - ip_addr = ipaddress.ip_address(int_data['dstAddr']) - ipv4 = '0.0.0.0' - ipv6 = '::' - if ip_addr.version == 4: - ipv4 = str(ip_addr) - else: - ipv6 = str(ip_addr) - attack_map_key = tps_utils.create_attack_hash( - mac=int_data['devMac'], port=int_data['dstPort'], ip_addr=ipv4, - ipv6_addr=ipv6) - logger.debug('Attack map key - [%s]', attack_map_key) - - if not self.drop_rpt_map.get(attack_map_key): - logger.debug( - 'Updating drop_rpt_map entry with zeros at time [%s] - [%s]', - strftime("%b-%d-%Y %H:%M:%S", localtime()), attack_map_key) - self.drop_rpt_map[attack_map_key] = (0, 0) - else: - logger.debug('Creating drop_rpt_map entry with key - [%s]', - attack_map_key) - tuple_val = self.drop_rpt_map[attack_map_key] - new_tuple = (tuple_val[0], 0) - self.drop_rpt_map[attack_map_key] = new_tuple - - if not self.count_map.get(attack_map_key): - self.count_map[attack_map_key] = list() - - curr_time = datetime.datetime.now() - self.count_map.get(attack_map_key).append((curr_time, pkt_bytes)) - time_bytes_tuple = self.count_map.get(attack_map_key) - count = 0 - total_bytes = 0 - for eval_time, pkt_bytes in time_bytes_tuple: - logger.debug('eval_time - [%s], pkt_bytes - [%s]', - eval_time, pkt_bytes) - delta = (curr_time - eval_time).total_seconds() - if delta > self.sample_interval: - time_bytes_tuple.remove((eval_time, pkt_bytes)) - else: - count += 1 - total_bytes += pkt_bytes - logger.info( - 'Total bytes [%s] and packet count [%s] received in window', - total_bytes, count) - if count > self.packet_count or total_bytes > self.byte_count: - logger.warning( - 'Attack detected at time [%s]- count [%s] & bytes [%s] ' - 'with key [%s]', - strftime("%b-%d-%Y %H:%M:%S", localtime()), - count, total_bytes, attack_map_key) - - attack_dict = dict( - src_mac=int_data['devMac'], - dst_port=int_data['dstPort'], - dst_ip=int_data['dstAddr']) - - # Send to SDN - last_attack = self.attack_map.get(attack_map_key) - if not last_attack or time.time() - last_attack > 1: - logger.info('Calling SDN, last attack sent - [%s]', - last_attack) - try: - self.attack_map[attack_map_key] = time.time() - self.attack_payload[attack_map_key] = attack_dict - self._send_attack(**attack_dict) - return True - except Exception as e: - logger.error('Unexpected error [%s]', e) - return False - else: - logger.debug( - 'Not calling SDN as last attack notification for %s' - ' was only %s seconds ago', - attack_dict, time.time() - last_attack) - return True - else: - logger.debug('No attack detected - count [%s]', count) - return False - - def process_drop_rpt(self, packet): - """ - Processes a drop report packet to determine if an attack stopped - :param packet: the packet to process - :return: T/F - True when an attack has been triggered - """ - ip_pkt, protocol, pkt_bytes = parse_ip_pkt(packet) - if ip_pkt and protocol and protocol == UDP_PROTO: - udp_packet = UDP(_pkt=ip_pkt.payload) - if udp_packet.dport == UDP_TRPT_DST_PORT: - hash_key, drop_count = extract_drop_rpt(udp_packet) - if self.drop_rpt_map.get(hash_key): - tuple_val = self.drop_rpt_map[hash_key] - old_count = tuple_val[0] - if old_count != drop_count: - new_tuple = (drop_count, 0) - self.drop_rpt_map[hash_key] = new_tuple - return False - else: - new_tuple = (tuple_val[0], tuple_val[1] + 1) - self.drop_rpt_map[hash_key] = new_tuple - if new_tuple[1] >= self.drop_count: - attack_body = self.attack_payload.get(hash_key) - if attack_body: - logger.info('Stopping attack with [%s]', - attack_body) - self._stop_attack(**attack_body) - - logger.info('Clearing maps after attacked stopped') - self.attack_payload[hash_key] = None - self.attack_map[hash_key] = None - self.count_map[hash_key] = None - self.drop_rpt_map[hash_key] = None - logger.info( - 'Cleared maps with hash [%s] at time [%s]', - hash_key, strftime("%b-%d-%Y %H:%M:%S", - localtime())) - return True - else: - self.drop_rpt_map[hash_key] = new_tuple - return False - return False - - -def parse_ip_pkt(packet): - ip_pkt = None - protocol = None - pkt_bytes = 0 - try: - if packet[Ether].type == IPV4_TYPE: - ip_pkt = IP(_pkt=packet[Ether].payload) - protocol = ip_pkt.proto - pkt_bytes = ip_pkt.len - elif packet[Ether].type == IPV6_TYPE: - ip_pkt = IPv6(_pkt=packet[Ether].payload) - protocol = ip_pkt.nh - pkt_bytes = ip_pkt.plen - except Exception as e: - logger.error('Unexpected error processing packet - [%s]', e) - - return ip_pkt, protocol, pkt_bytes - - -class IntLoggerAE(PacketAnalytics): - """ - Logs only INT packets - """ - - def process_packet(self, packet, udp_dport=UDP_INT_DST_PORT): - """ - Logs the INT data within the packet - :param packet: the INT packet - :param udp_dport: the UDP port value on which to filter - :return: False - """ - logger.info('INT Packet data - [%s]', extract_int_data(packet[Ether])) - return False - - def process_drop_rpt(self, packet): - pass - - -class LoggerAE(PacketAnalytics): - """ - Logging only - """ - - def handle_packet(self, packet, ip_proto=None): - """ - Logs every received packet's summary data - :param packet: extracts data from here - :param ip_proto: does nothing here - :return: False - """ - logger.info('Packet data - [%s]', packet.summary()) - return False - - def process_packet(self, packet, udp_dport=UDP_INT_DST_PORT): - """ - No need to implement - :param packet: the packet that'll never come in - :param udp_dport: the UDP port value on which to filter - :raises NotImplemented - """ - raise NotImplemented - - def process_drop_rpt(self, packet): - pass diff --git a/trans_sec/controller/aggregate_controller.py b/trans_sec/controller/aggregate_controller.py index 365e5570..d4c0cba7 100755 --- a/trans_sec/controller/aggregate_controller.py +++ b/trans_sec/controller/aggregate_controller.py @@ -17,21 +17,8 @@ logger = getLogger('aggregate_controller') -try: - from trans_sec.p4runtime_lib.aggregate_switch import ( - AggregateSwitch as P4RTSwitch) -except Exception as e: - logger.warning( - "Error [%s] - while attempting to import " - "trans_sec.p4runtime_lib.aggregate_switch.AggregateSwitch", e) - -try: - from trans_sec.bfruntime_lib.aggregate_switch import ( +from trans_sec.bfruntime_lib.aggregate_switch import ( AggregateSwitch as BFRTSwitch) -except Exception as e: - logger.warning( - "Error [%s] - while attempting to import " - "trans_sec.bfruntime_lib.aggregate_switch.AggregateSwitch", e) class AggregateController(AbstractController): @@ -48,14 +35,8 @@ def __init__(self, platform, p4_build_out, topo, log_dir, load_p4=True): self.attack_dict = {} def instantiate_switch(self, sw_info): - if 'arch' in sw_info and sw_info['arch'] == 'tna': - logger.info('Instantiating BFRT AggregateSwitch') - return BFRTSwitch(sw_info=sw_info) - else: - return P4RTSwitch( - sw_info=sw_info, - proto_dump_file='{}/{}-switch-controller.log'.format( - self.log_dir, sw_info['name'])) + logger.info('Instantiating BFRT AggregateSwitch') + return BFRTSwitch(sw_info=sw_info) def __get_agg_switch(self): return self.switches[0] diff --git a/trans_sec/controller/appcontroller.py b/trans_sec/controller/appcontroller.py deleted file mode 100644 index e17350b2..00000000 --- a/trans_sec/controller/appcontroller.py +++ /dev/null @@ -1,127 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import logging -import subprocess - -from trans_sec.controller.shortest_path import ShortestPath - -logger = logging.getLogger('appcontroller') - - -class AppController: - - def __init__(self, manifest=None, target=None, topo=None, net=None, - links=None): - self.manifest = manifest - self.target = target - self.conf = manifest['targets'][target] - self.topo = topo - self.net = net - self.links = links - - @staticmethod - def read_entries(filename): - entries = [] - with open(filename, 'r') as f: - for line in f: - line = line.strip() - if line == '': - continue - entries.append(line) - return entries - - @staticmethod - def add_entries(thrift_port=9090, sw=None, entries=None): - assert entries - if sw: - thrift_port = sw.thrift_port - - p = subprocess.Popen( - ['simple_switch_CLI', '--thrift-port', str(thrift_port)], - stdin=subprocess.PIPE) - p.communicate(input='\n'.join(entries)) - - @staticmethod - def read_register(register, idx, thrift_port=9090, sw=None): - if sw: - thrift_port = sw.thrift_port - p = subprocess.Popen( - ['simple_switch_CLI', '--thrift-port', str(thrift_port)], - stdin=subprocess.PIPE, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - stdout, stderr = p.communicate( - input="register_read %s %d" % (register, idx)) - reg_val = list(filter(lambda l: ' %s[%d]' % (register, idx) in l, - stdout.split('\n')))[0].split('= ', 1)[1] - return int(reg_val) - - def start(self): - shortest_path = ShortestPath(self.links) - entries = {} - for sw in self.topo.switches(): - entries[sw] = [] - if ('switches' in self.conf - and sw in self.conf['switches'] - and 'entries' in self.conf['switches'][sw]): - extra_entries = self.conf['switches'][sw]['entries'] - if type(extra_entries) == list: # array of entries - entries[sw] += extra_entries - else: # path to file that contains entries - entries[sw] += self.read_entries(extra_entries) - - for host_name in self.topo.host_links: - host = self.net.get(host_name) - for link in self.topo.host_links[host_name].values(): - iface = host.intfNames()[link['idx']] - # use mininet to set ip and mac to let it know the change - host.setIP(link['host_ip'], 24) - host.setMAC(link['host_mac']) - host.cmd('arp -i %s -s %s %s' % ( - iface, link['sw_ip'], link['sw_mac'])) - host.cmd('ethtool --offload %s rx off tx off' % iface) - host.cmd('ip route add %s dev %s' % (link['sw_ip'], iface)) - - # TODO - determine why this was outside of the for block? - host.setDefaultRoute("via %s" % link['sw_ip']) - - for host in self.net.hosts: - for sw in self.net.switches: - path = shortest_path.get(sw.name, host.name, - exclude=lambda n: n[0] == 'h') - if not path: - continue - if not path[1][0] == 's': - continue # next hop is a switch - - for h2 in self.net.hosts: - if host == h2: - continue - path = shortest_path.get(host.name, h2.name, - exclude=lambda n: n[0] == 'h') - if not path: - continue - h_link = self.topo.host_links[host.name][path[1]] - h2_link = self.topo.host_links[h2.name].values()[0] - host.cmd('ip route add %s via %s' % ( - h2_link['host_ip'], h_link['sw_ip'])) - - logger.info("Configuring entries in p4 tables") - for sw_name in entries: - logger.info("Configuring switch... %s", sw_name) - sw = self.net.get(sw_name) - if entries[sw_name]: - self.add_entries(sw=sw, entries=entries[sw_name]) - logger.info("Configuration complete.") - - def stop(self): - pass diff --git a/trans_sec/controller/core_controller.py b/trans_sec/controller/core_controller.py index 889792b1..fcba4747 100755 --- a/trans_sec/controller/core_controller.py +++ b/trans_sec/controller/core_controller.py @@ -15,24 +15,10 @@ from trans_sec.controller.abstract_controller import AbstractController from trans_sec.controller.ddos_sdn_controller import CORE_CTRL_KEY -logger = getLogger('core_controller') +from trans_sec.bfruntime_lib.core_switch import (CoreSwitch as BFRTSwitch) -try: - logger.debug('Importing CoreSwitch as P4RTSwitch') - from trans_sec.p4runtime_lib.core_switch import CoreSwitch as P4RTSwitch - logger.debug('Imported CoreSwitch as P4RTSwitch') -except Exception as e: - logger.warning( - "Error [%s] - while attempting to import " - "trans_sec.p4runtime_lib.core_switch.CoreSwitch", e) -try: - logger.debug('Importing CoreSwitch as BFRTSwitch') - from trans_sec.bfruntime_lib.core_switch import ( - CoreSwitch as BFRTSwitch) - logger.debug('Imported CoreSwitch as BFRTSwitch') -except Exception as e: - logger.warning('Could not import bfrt classes') +logger = getLogger('core_controller') class CoreController(AbstractController): @@ -46,14 +32,8 @@ def __init__(self, platform, p4_build_out, topo, log_dir, load_p4=True): def instantiate_switch(self, sw_info): logger.info('Instantiating switch with arch - [%s]', sw_info) - if 'arch' in sw_info and sw_info['arch'] == 'tna': - logger.info('Instantiating BFRT CoreSwitch') - return BFRTSwitch(sw_info=sw_info) - else: - return P4RTSwitch( - sw_info=sw_info, - proto_dump_file='{}/{}-switch-controller.log'.format( - self.log_dir, sw_info['name'])) + logger.info('Instantiating BFRT CoreSwitch') + return BFRTSwitch(sw_info=sw_info) def __get_core_switch(self): return self.switches[0] diff --git a/trans_sec/controller/ddos_sdn_controller.py b/trans_sec/controller/ddos_sdn_controller.py index 4b3e8c1e..7daaa376 100755 --- a/trans_sec/controller/ddos_sdn_controller.py +++ b/trans_sec/controller/ddos_sdn_controller.py @@ -36,7 +36,6 @@ AGG_CTRL_KEY = 'aggregate' CORE_CTRL_KEY = 'core' -GATEWAY_CTRL_KEY = 'gateway' class DdosSdnController: @@ -228,35 +227,6 @@ def __data_inspection(self, di_req, del_flag=False): logger.warning('Could not find switch with device_id - [%s]', di_req['device_id']) - def remove_attacker(self, attack): - """ - Removes a device to mitigate an attack - :param attack: dict of attack - """ - host, gw_controller = self.__get_attack_host(attack) - logger.info('Adding attack to gateways with host - [%s]', host) - try: - gw_controller.remove_attacker(attack, host) - except Exception as e: - logger.error( - 'Error removing attacker to host - [%s] with error - [%s])', - host, e) - - def add_attacker(self, attack): - """ - Adds a device to mitigate an attack - :param attack: dict of attack - """ - host, gw_controller = self.__get_attack_host(attack) - logger.info('Adding attack to gateways with host - [%s]', host) - try: - gw_controller.add_attacker(attack, host) - except Exception as e: - logger.error( - 'Error adding gateway attacker to host - [%s] ' - 'with error - [%s])', host, e) - raise e - def remove_agg_attacker(self, attack): """ Removes a device to mitigate an attack @@ -386,38 +356,6 @@ def get_core_controller(self): core_controller = self.controllers.get(CORE_CTRL_KEY) return core_controller - def __get_attack_host(self, attack): - """ - Returns the host value or None - :param attack: - :return: - """ - gateway_controller = self.controllers.get(GATEWAY_CTRL_KEY) - if gateway_controller: - logger.info('Attack received - %s', attack) - - conditions = {'mac': attack['src_mac']} - logger.debug('Created conditions - [%s]', conditions) - values = self.topo.get('hosts').values() - logger.debug('Creating host with values - [%s]', values) - host = list(filter( - lambda item: all( - (item[k] == v for (k, v) in conditions.items())), - values)) - - logger.debug( - 'Check the hosts and register the attack with host object ' - '- [%s]', host) - logger.debug('host.__class__ - [%s]', host.__class__) - if len(host) > 0: - logger.debug('host len is - [%s]', len(host)) - return host[0], gateway_controller - else: - logger.error('No Device Matches MAC [%s]', - attack.get('src_mac')) - else: - logger.warning('No Gateway Controller call') - def __main_loop(self): """ Starts polling thread/Error adding attacker to host diff --git a/trans_sec/controller/gateway_controller.py b/trans_sec/controller/gateway_controller.py deleted file mode 100755 index 38a29fcd..00000000 --- a/trans_sec/controller/gateway_controller.py +++ /dev/null @@ -1,143 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import ipaddress -from logging import getLogger -from trans_sec.controller.abstract_controller import AbstractController -from trans_sec.controller.ddos_sdn_controller import GATEWAY_CTRL_KEY - -logger = getLogger('gateway_controller') - -try: - from trans_sec.p4runtime_lib.gateway_switch import ( - GatewaySwitch as P4RTSwitch) -except Exception as e: - logger.warning( - "Error [%s] - while attempting to import " - "trans_sec.p4runtime_lib.aggregate_switch.AggregateSwitch", e) - -try: - from trans_sec.bfruntime_lib.gateway_switch import ( - GatewaySwitch as BFRTSwitch) -except Exception as e: - logger.warning( - "Error [%s] - while attempting to import " - "trans_sec.bfruntime_lib.aggregate_switch.AggregateSwitch", e) - -try: - from trans_sec.p4runtime_lib.gateway_switch import ( - GatewaySwitch as P4RTSwitch) -except Exception as e: - logger.warning( - "Error [%s] - while attempting to import " - "trans_sec.p4runtime_lib.gateway_switch.GatewaySwitch", e) - -try: - from trans_sec.bfruntime_lib.gateway_switch import ( - GatewaySwitch as BFRTSwitch) -except Exception as e: - logger.warning('Could not import bfrt classes') - - -class GatewayController(AbstractController): - """ - Implementation of the controller for a switch running the gateway.p4 - program - """ - def __init__(self, platform, p4_build_out, topo, log_dir, load_p4=True): - super(self.__class__, self).__init__( - platform, p4_build_out, topo, GATEWAY_CTRL_KEY, log_dir, load_p4) - - def instantiate_switch(self, sw_info): - if 'arch' in sw_info and sw_info['arch'] == 'tna': - return BFRTSwitch(sw_info=sw_info) - else: - return P4RTSwitch( - sw_info=sw_info, - proto_dump_file='{}/{}-switch-controller.log'.format( - self.log_dir, sw_info['name'])) - - def make_rules(self, sw, north_facing_links, south_facing_links, - add_di): - """ - Overrides the abstract method from super - :param sw: switch object - :param north_facing_links: northbound links - :param south_facing_links: southbound links - :param add_di: when True inserts into the data_inspection_t table - """ - for device_link in south_facing_links: - device = self.topo['hosts'].get(device_link['south_node']) - - if device: - logger.info('Gateway: ' + sw.name + - ' connects to Device: ' + device['name'] + - ' on physical port ' + - str(device_link.get('south_facing_port')) + - ' to IP ' + device.get('ip') + - ':' + str(device.get('ip_port'))) - - if add_di: - sw.add_data_inspection(dev_id=device['id'], - dev_mac=device['mac']) - - def __process_gw_attack(self, attack, host): - attack_switch = None - for switch in self.switches: - if switch.type == 'gateway': - di_match_mac = switch.get_data_inspection_src_mac_keys() - if len(di_match_mac) > 0: - logger.debug( - 'Data inspection table keys on device [%s] - [%s]', - switch.sw_info['id'], di_match_mac) - if attack['src_mac'] in di_match_mac: - logger.info('Found source switch - [%s]', switch.name) - attack_switch = switch - else: - attack_switch = self.switches[0] - break - if attack_switch: - logger.info('Adding an attack [%s] to host [%s] and switch [%s]', - attack, host, attack_switch.name) - ip_addr = ipaddress.ip_address(attack['src_ip']) - logger.info('Attack ip addr - [%s]', ip_addr) - logger.debug('Attack ip addr class - [%s]', ip_addr.__class__) - if ip_addr.version == 6: - logger.debug('Attack is IPv6') - proto_key = 'ipv6' - else: - logger.debug('Attack is IPv4') - proto_key = 'ipv4' - - dst_addr_key = 'hdr.{}.dstAddr'.format(proto_key) - return attack_switch, dst_addr_key - else: - return None - - def add_attacker(self, attack, host): - logger.info('Attack received by the controller of type [%s] - [%s]', - self.switch_type, attack) - attack_switch, dst_addr_key = self.__process_gw_attack(attack, host) - if attack_switch and dst_addr_key: - attack['host'] = host - attack[dst_addr_key] = attack['src_ip'] - attack_switch.add_attack(**attack) - - def remove_attacker(self, attack, host): - attack_switch, dst_addr_key = self.__process_gw_attack(attack, host) - if attack_switch and dst_addr_key: - attack['host'] = host - attack[dst_addr_key] = attack['src_ip'] - attack_switch.stop_attack(**attack) - - def count_dropped_packets(self): - pass diff --git a/trans_sec/controller/http_server_flask.py b/trans_sec/controller/http_server_flask.py index 929e3337..17c82b9d 100644 --- a/trans_sec/controller/http_server_flask.py +++ b/trans_sec/controller/http_server_flask.py @@ -49,11 +49,6 @@ def start(self): DataInspection, '/dataInspection', resource_class_kwargs={'sdn_controller': self.sdn_controller}) - logger.info('Starting gwAttack') - self.api.add_resource( - GwAttack, '/gwAttack', - resource_class_kwargs={'sdn_controller': self.sdn_controller}) - logger.info('Starting aggAttack') self.api.add_resource( AggAttack, '/aggAttack', @@ -187,48 +182,6 @@ def delete(self): return json.dumps({"success": True}), 201 -class GwAttack(Resource): - """ - Class for exposing web service to issue an attack call to gateway.p4 - """ - - parser = reqparse.RequestParser() - parser.add_argument('src_mac', type=str) - parser.add_argument('src_ip', type=str) - parser.add_argument('dst_ip', type=str) - parser.add_argument('dst_port', type=str) - parser.add_argument('packet_size', type=str) - parser.add_argument('attack_type', type=str) - - def __init__(self, **kwargs): - logger.info('Starting GwAttack context') - self.sdn_controller = kwargs['sdn_controller'] - - @swagger.tags(['gatewayAttackStart']) - @swagger.response(response_code=201, - description='Mitigated attack on gateway') - @swagger.reqparser(name='GwAttackParser', parser=parser) - def post(self): - logger.info('Gateway attack requested') - args = self.parser.parse_args() - - logger.info('args - [%s]', args) - self.sdn_controller.add_attacker(args) - return json.dumps({"success": True}), 201 - - @swagger.tags(['gatewayAttackStop']) - @swagger.response(response_code=201, - description='Unmitigated attacks from gateway') - @swagger.reqparser(name='GwAttackParser', parser=parser) - def delete(self): - logger.info('GW Attacker to remove') - args = self.parser.parse_args() - - logger.info('args - [%s]', args) - self.sdn_controller.remove_attacker(args) - return json.dumps({"success": True}), 201 - - class AggDataForward(Resource): """ Class for exposing web service to enter a data_forward entry into the P4 diff --git a/trans_sec/device_software/hulk-attack.sh b/trans_sec/device_software/hulk-attack.sh deleted file mode 100755 index afe4b119..00000000 --- a/trans_sec/device_software/hulk-attack.sh +++ /dev/null @@ -1,24 +0,0 @@ - #!/bin/bash - -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -PY_SCRIPT=$1 -HOST=$2 -DURATION=$3 - -python $PY_SCRIPT http://${HOST} & -HULK_PID=$(echo $!) -sleep $DURATION -kill -9 $HULK_PID -echo "-- HULK Attack Terminated --" diff --git a/trans_sec/mininet/__init__.py b/trans_sec/mininet/__init__.py deleted file mode 100644 index 77c9835a..00000000 --- a/trans_sec/mininet/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/trans_sec/mininet/apptopo.py b/trans_sec/mininet/apptopo.py deleted file mode 100644 index b58e4d0d..00000000 --- a/trans_sec/mininet/apptopo.py +++ /dev/null @@ -1,89 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from mininet.topo import Topo - - -class AppTopo(Topo): - - def __init__(self, links, latencies, log_dir="/tmp", bws=None, **opts): - Topo.__init__(self, **opts) - - nodes = sum(map(list, zip(*links)), []) - host_names = sorted(list(set(filter(lambda n: n[0] == 'h', nodes)))) - sw_names = sorted(list(set(filter(lambda n: n[0] == 's', nodes)))) - sw_ports = dict([(sw, []) for sw in sw_names]) - - self.host_links = dict() - self.sw_links = dict([(sw, {}) for sw in sw_names]) - - for sw_name in sw_names: - self.addSwitch(sw_name, log_file="%s/%s.log" % (log_dir, sw_name)) - - for host_name in host_names: - host_num = int(host_name[1:]) - - self.addHost(host_name) - - self.host_links[host_name] = {} - host_links = filter( - lambda l: l[0] == host_name or l[1] == host_name, links) - - sw_idx = 0 - for link in host_links: - sw = link[0] if link[0] != host_name else link[1] - sw_num = int(sw[1:]) - assert sw[0] ==\ - 's', "Hosts should be connected to switches, "\ - "not {}".format(str(sw)) - host_ip = "10.0.%d.%d" % (sw_num, host_num) - host_mac = '00:00:00:00:%02x:%02x' % (sw_num, host_num) - delay_key = ''.join([host_name, sw]) - delay = latencies[ - delay_key] if delay_key in latencies else '0ms' - bw = bws[delay_key] if delay_key in bws else None - sw_ports[sw].append(host_name) - self.host_links[host_name][sw] = dict( - idx=sw_idx, - host_mac=host_mac, - host_ip=host_ip, - sw=sw, - sw_mac="00:00:00:00:%02x:%02x" % (sw_num, host_num), - sw_ip="10.0.%d.%d" % (sw_num, 254), - sw_port=sw_ports[sw].index(host_name) + 1 - ) - self.addLink(host_name, sw, delay=delay, bw=bw, - addr1=host_mac, - addr2=self.host_links[host_name][sw]['sw_mac']) - sw_idx += 1 - - for link in links: # only check switch-switch links - sw1, sw2 = link - if sw1[0] != 's' or sw2[0] != 's': - continue - - delay_key = ''.join(sorted([sw1, sw2])) - delay = latencies[delay_key] if delay_key in latencies else '0ms' - bw = bws[delay_key] if delay_key in bws else None - - self.addLink(sw1, sw2, delay=delay, bw=bw) # , max_queue_size=10) - sw_ports[sw1].append(sw2) - sw_ports[sw2].append(sw1) - - sw1_num, sw2_num = int(sw1[1:]), int(sw2[1:]) - sw1_port = dict(mac="00:00:00:%02x:%02x:00" % (sw1_num, sw2_num), - port=sw_ports[sw1].index(sw2) + 1) - sw2_port = dict(mac="00:00:00:%02x:%02x:00" % (sw2_num, sw1_num), - port=sw_ports[sw2].index(sw1) + 1) - - self.sw_links[sw1][sw2] = [sw1_port, sw2_port] - self.sw_links[sw2][sw1] = [sw2_port, sw1_port] diff --git a/trans_sec/mininet/exercise.py b/trans_sec/mininet/exercise.py deleted file mode 100644 index f0f5567e..00000000 --- a/trans_sec/mininet/exercise.py +++ /dev/null @@ -1,343 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Unit tests for convert.py -import logging -import os - -from mininet.cli import CLI -from mininet.link import TCLink, Intf -from mininet.net import Mininet -from mininet.topo import Topo - -from trans_sec.mininet.p4_mininet import P4Host -from trans_sec.p4runtime_lib.p4runtime_switch import MininetSwitch - -logger = logging.getLogger('exercise') - - -class ExerciseTopo(Topo): - """ - The mininet topology class for the P4 tutorial exercises. - A custom class is used because the exercises make a few topology - assumptions, mostly about the IP and MAC addresses. - """ - - def __init__(self, hosts, switches, external, links, log_dir, **opts): - Topo.__init__(self, **opts) - self.sw_port_mapping = {} - - for sw_name, sw in switches.items(): - dpid = self.__int_to_dpid(sw['id']) - self.addSwitch(sw['name'], dpid=dpid, - log_file="%s/%s.log" % (log_dir, sw)) - - for link in links: - np = link.get('north_facing_port') - sp = link.get('south_facing_port') - - # Two switches - if np and sp: - s_switch = switches.get(link.get('south_node')) - n_switch = switches.get(link.get('north_node')) - - self.addLink(s_switch['name'], n_switch['name'], - delay=link['latency'], bw=link['bandwidth'], - addr1=s_switch['mac'], - addr2=n_switch['mac']) - self.__add_switch_port(s_switch['name'], - n_switch['name'], - link['north_facing_port']) - self.__add_switch_port(n_switch['name'], - s_switch['name'], - link['south_facing_port']) - logger.info('Adding Switch Link %s %s port:%d <--> port:%d' % ( - link['south_node'], link['north_node'], - link['north_facing_port'], link['south_facing_port'])) - - # South switch has a north facing port - elif np: - s_switch = switches.get(link.get('south_node')) - n_host = hosts.get(link.get('north_node')) - if n_host is None: - n_host = external.get(link.get('north_node')) - # ignore externals - if n_host is not None: - self.addHost(n_host['name'], - ip=n_host['ip'] + '/24', - mac=n_host['mac']) - self.addLink(n_host['name'], s_switch['name'], - delay=link['latency'], bw=link['bandwidth'], - addr1=n_host['mac'], - addr2=s_switch['mac']) - self.__add_switch_port(s_switch['name'], - n_host['name'], np) - logger.info( - "Adding host %s link %s %s to switch %s %s on port %s", - n_host.get('name'), n_host.get('ip'), - n_host.get('mac'), s_switch.get('name'), - s_switch.get('mac'), np) - # North switch has a south facing port to the host - elif sp is not None: - n_switch = switches.get(link.get('north_node')) - s_host = hosts.get(link.get('south_node')) - - # ignore externals - if s_host is not None: - self.addHost(s_host['name'], - ip=s_host['ip'] + '/24', - mac=s_host['mac']) - if link.get('south_facing_mac'): - self.addLink(s_host['name'], n_switch['name'], - delay=link['latency'], - bw=link['bandwidth'], - addr1=s_host['mac'], - addr2=link.get('south_facing_mac')) - self.__add_switch_port(n_switch['name'], - s_host['name'], sp) - logger.info("Adding host %s link %s %s to switch %s %s" - " on port %d", - s_host.get('name'), s_host.get('ip'), - s_host.get('mac'), n_switch.get('name'), - link.get('south_facing_port'), sp,) - else: - self.addLink(s_host['name'], n_switch['name'], - delay=link['latency'], - bw=link['bandwidth'], - addr1=s_host['mac'], - addr2=n_switch['mac']) - self.__add_switch_port(n_switch['name'], - s_host['name'], sp) - logger.info("Adding host %s link %s %s to switch %s %s" - " on port %d", - s_host.get('name'), s_host.get('ip'), - s_host.get('mac'), n_switch.get('name'), - n_switch['mac'], sp,) - else: - logger.info('Error in link. At least one port must be ' - 'defined %s', link) - - self.__print_port_mapping() - - @staticmethod - def __int_to_dpid(dpid): - try: - dpid = hex(dpid)[2:] - dpid = '0' * (16 - len(dpid)) + dpid - return dpid - except IndexError: - raise Exception('Unable to derive default data path ID - ' - 'please either specify a dpid or use a ' - 'canonical switch name such as s23.') - - def __add_switch_port(self, sw, target, port=None): - if sw not in self.sw_port_mapping: - self.sw_port_mapping[sw] = [] - if port is None: - port_num = len(self.sw_port_mapping[sw]) + 1 - self.sw_port_mapping[sw].append((port_num, target)) - else: - self.sw_port_mapping[sw].append((port, target)) - - def __print_port_mapping(self): - logger.info("Switch port mapping:") - for sw in sorted(self.sw_port_mapping.keys()): - logger.info("%s: " % sw,) - for port_num, target in self.sw_port_mapping[sw]: - logger.info("%d:%s\t" % (port_num, target),) - - -class ExerciseRunner: - """ - Attributes: - log_dir : string // directory for mininet log files - pcap_dir : string // directory for mininet switch pcap files - hosts : list // list of mininet host names - switches : dict // mininet host names and their - associated properties - links : list // list of mininet link properties - switch_json : string // json of the compiled p4 example - topo : Topo object // The mininet topology instance - mininet : Mininet object // The mininet instance - """ - - def __init__(self, topo, log_dir, pcap_dir, switch_json, start_cli=False): - """ Initializes some attributes and reads the topology json. Does not - actually run the exercise. Use run_exercise() for that. - - Arguments: - topo : dict // A dict describing the mininet topology - log_dir : string // Path to a directory for storing - exercise logs - pcap_dir : string // Same for mininet switch pcap files - switch_json : string // Path to a compiled p4 json for bmv2 - """ - - logger.info('Instantiating P4 Mininet exercise with topology - [%s]', - topo) - self.hosts = topo['hosts'] - self.switches = topo['switches'] - self.external = topo.get('external') - self.links = topo['links'] - self.start_cli = start_cli - - # Ensure all the needed directories exist and are directories - for dir_name in [log_dir, pcap_dir]: - if dir_name and not os.path.isdir(dir_name): - if os.path.exists(dir_name): - raise Exception( - "'%s' exists and is not a directory!" % dir_name) - os.mkdir(dir_name) - self.log_dir = log_dir - self.pcap_dir = pcap_dir - self.switch_json = switch_json - self.topo = ExerciseTopo(self.hosts, self.switches, self.external, - self.links, self.log_dir) - self.mininet = self.__setup_mininet() - self.running = False - self.fwd_runner = None - - def run_exercise(self): - """ Sets up the mininet instance, programs the switches, - and starts the mininet CLI. This is the main method to run after - initializing the object. - """ - # Initialize mininet with the topology specified by the config - logger.info('Running exercise') - self.__add_external_connections() - self.mininet.start() - - logger.info('Programming mininet hosts') - self.__program_hosts() - - if self.start_cli: - logger.info('Starting mininet CLI') - self.do_net_cli() - - logger.info('Completed mininet setup') - self.running = True - - def stop(self): - if self.fwd_runner: - self.fwd_runner.stop() - self.running = False - - def __setup_mininet(self): - """ Create the mininet network object, and store it as self.mininet. - - Side effects: - - Mininet topology instance stored as self.topo - - Mininet instance stored as self.mininet - """ - switch_class = self.__configure_p4_switch() - - logger.info('Starting mininet with topology - [%s]', self.topo) - return Mininet( - topo=self.topo, link=TCLink, host=P4Host, switch=switch_class, - autoPinCpus=True) - - def __configure_p4_switch(self): - """ - Returns a P4RuntimeSwitch class for use by Mininet - """ - if self.pcap_dir: - pcap_dump_dir = self.pcap_dir - else: - pcap_dump_dir = self.log_dir - - switch_args = { - 'sw_path': 'simple_switch_grpc', - 'json_path': self.switch_json, - 'log_console': True, - 'pcap_dump': pcap_dump_dir, - } - - class ConfiguredMininetSwitch(MininetSwitch): - def __init__(self, *opts, **kwargs): - kwargs.update(switch_args) - MininetSwitch.__init__(self, *opts, **kwargs) - - def describe(self): - logger.info("%s -> gRPC port: %s", self.name, - self.grpc_port) - - return ConfiguredMininetSwitch - - def __add_external_connections(self): - if self.external: - logger.info('External dict contents - [%s]', self.external) - for link in self.links: - external = self.external.get(link['north_node']) - if external is not None: - sw_obj = self.mininet.get(link['south_node']) - Intf(external['id'], node=sw_obj) - - def __program_hosts(self): - logger.info('Programming hosts with values - [%s]', self.hosts) - for name, host in self.hosts.items(): - logger.debug('Programming host - [%s]', host) - h = self.mininet.get(host['name']) - h_iface = list(h.intfs.values())[0] - link = h_iface.link - - sw_iface = link.intf1 if link.intf1 != h_iface else link.intf2 - sw_ip = host['switch_ip'] - - # Ensure each host's interface name is unique, or else - # mininet cannot shutdown gracefully - h.defaultIntf().rename('%s-eth0' % host['name']) - # static arp entries and default routes - h.cmd('arp -i %s -s %s %s' % (h_iface.name, sw_ip, sw_iface.mac)) - h.cmd('ethtool --offload %s rx off tx off' % h_iface.name) - h.cmd('route add default gw %s %s' % (sw_ip, h_iface.name)) - h.cmd('/usr/sbin/sshd -D&') - h.setDefaultRoute("via %s" % sw_ip) - - def do_net_cli(self): - """ Starts up the mininet CLI and prints some helpful output. - - Assumes: - - A mininet instance is stored as self.mininet and - self.mininet.start() has been called. - """ - for s in self.mininet.switches: - s.describe() - for h in self.mininet.hosts: - try: - h.describe() - except Exception as e: - logger.warning("Ignore exception [%s]", e) - logger.info("Starting mininet CLI") - # Generate a message that will be printed by the Mininet CLI to make - # interacting with the simple switch a little easier. - print('') - print('==============================================================') - print('Welcome to the BMV2 Mininet CLI!') - print('==============================================================') - print('Your P4 program is installed into the BMV2 software switch') - print('and your initial configuration is loaded. You can interact') - print('with the network using the mininet CLI below.') - print('') - if self.switch_json: - print('To inspect or change the switch configuration, connect to') - print('CLI from your host operating system using this command:') - print(' simple_switch_CLI --thrift-port ') - print('') - print('To view a switch log, run this command from your host OS:') - print(' tail -f %s/.log' % self.log_dir) - print('') - print('To view the switch output pcap, check the pcap files in %s:' - % self.pcap_dir) - print(' for example run: sudo tcpdump -xxx -r s1-eth1.pcap') - print('') - - CLI(self.mininet) diff --git a/trans_sec/mininet/p4_mininet.py b/trans_sec/mininet/p4_mininet.py deleted file mode 100644 index 3c240f2b..00000000 --- a/trans_sec/mininet/p4_mininet.py +++ /dev/null @@ -1,159 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -import os -import tempfile -from sys import exit -from time import sleep - -from mininet.log import info, error, debug -from mininet.moduledeps import pathCheck -from mininet.node import Switch, Host - -from trans_sec.utils.netstat import check_listening_on_port - -SWITCH_START_TIMEOUT = 10 # seconds - -logger = logging.getLogger('p4_mininet') - - -class P4Host(Host): - def config(self, **params): - r = super(Host, self).config(**params) - - self.defaultIntf().rename("eth0") - - for off in ["rx", "tx", "sg"]: - cmd = "/sbin/ethtool --offload eth0 %s off" % off - self.cmd(cmd) - - # disable IPv6 - self.cmd("sysctl -w net.ipv6.conf.all.disable_ipv6=1") - self.cmd("sysctl -w net.ipv6.conf.default.disable_ipv6=1") - self.cmd("sysctl -w net.ipv6.conf.lo.disable_ipv6=1") - - return r - - def describe(self): - logger.info("Name: %s, dflt intf: %s/%s/%s", - self.name, - self.defaultIntf().name, - self.defaultIntf().IP(), - self.defaultIntf().MAC()) - - -class P4Switch(Switch): - """P4 virtual switch""" - device_id = 0 - - def __init__(self, name, sw_path=None, json_path=None, - thrift_port=None, - pcap_dump=False, - log_console=False, - verbose=False, - device_id=None, - enable_debugger=False, - **kwargs): - Switch.__init__(self, name, **kwargs) - logger.info('The other P4Switch') - assert sw_path - assert json_path - # make sure that the provided sw_path is valid - pathCheck(sw_path) - # make sure that the provided JSON file exists - if not os.path.isfile(json_path): - error("Invalid JSON file.\n") - exit(1) - self.sw_path = sw_path - self.json_path = json_path - self.verbose = verbose - logfile = "/tmp/p4s.{}.log".format(self.name) - self.output = open(logfile, 'w') - self.thrift_port = thrift_port - if check_listening_on_port(self.thrift_port): - error('%s cannot bind port %d because it is bound by another ' - 'process\n' % (self.name, self.thrift_port)) - exit(1) - self.pcap_dump = pcap_dump - self.enable_debugger = enable_debugger - self.log_console = log_console - if device_id is not None: - self.device_id = device_id - P4Switch.device_id = max(P4Switch.device_id, device_id) - else: - self.device_id = P4Switch.device_id - P4Switch.device_id += 1 - self.nanomsg = "ipc:///tmp/bm-{}-log.ipc".format(self.device_id) - - @classmethod - def setup(cls): - pass - - def check_switch_started(self, pid): - """While the process is running (pid exists), we check if the Thrift - server has been started. If the Thrift server is ready, we assume that - the switch was started successfully. This is only reliable if the - Thrift server is started at the end of the init process""" - while True: - if not os.path.exists(os.path.join("/proc", str(pid))): - return False - if check_listening_on_port(self.thrift_port): - return True - sleep(0.5) - - def start(self, controllers): - """ - Start up a new P4 switch - """ - info("Starting P4 switch {}.\n".format(self.name)) - args = [self.sw_path] - for port, intf in self.intfs.items(): - if not intf.IP(): - args.extend(['-i', str(port) + "@" + intf.name]) - if self.pcap_dump: - args.append("--pcap pcaps") - # args.append("--useFiles") - if self.thrift_port: - args.extend(['--thrift-port', str(self.thrift_port)]) - if self.nanomsg: - args.extend(['--nanolog', self.nanomsg]) - args.extend(['--device-id', str(self.device_id)]) - P4Switch.device_id += 1 - args.append(self.json_path) - if self.enable_debugger: - args.append("--debugger") - if self.log_console: - args.append("--log-console") - logfile = "/tmp/p4s.{}.log".format(self.name) - info(' '.join(args) + "\n") - - with tempfile.NamedTemporaryFile() as f: - self.cmd(' '.join( - args) + ' >' + logfile + ' 2>&1 & echo $! >> ' + f.name) - pid = int(f.read()) - debug("P4 switch {} PID is {}.\n".format(self.name, pid)) - if not self.check_switch_started(pid): - error("P4 switch {} did not start correctly.\n".format(self.name)) - exit(1) - info("P4 switch {} has been started.\n".format(self.name)) - - def stop(self, delete_interfaces=True): - """ - Terminate P4 switch. - """ - self.output.flush() - self.cmd('kill %' + self.sw_path) - self.cmd('wait') - self.deleteIntfs() - super(P4Switch, self).stop(delete_interfaces) diff --git a/trans_sec/p4runtime_lib/__init__.py b/trans_sec/p4runtime_lib/__init__.py deleted file mode 100644 index 77c9835a..00000000 --- a/trans_sec/p4runtime_lib/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/trans_sec/p4runtime_lib/aggregate_switch.py b/trans_sec/p4runtime_lib/aggregate_switch.py deleted file mode 100644 index f41d2601..00000000 --- a/trans_sec/p4runtime_lib/aggregate_switch.py +++ /dev/null @@ -1,148 +0,0 @@ -# Copyright (c) 2020 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Originally copied from: -# -# Copyright 2017-present Open Networking Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import logging - -from trans_sec.p4runtime_lib.p4rt_switch import P4RuntimeSwitch -from trans_sec.consts import UDP_INT_DST_PORT -from trans_sec.controller.ddos_sdn_controller import AGG_CTRL_KEY - -logger = logging.getLogger('aggregate_switch') - - -class AggregateSwitch(P4RuntimeSwitch): - def __init__(self, sw_info, proto_dump_file=None): - """ - Construct Switch class to control BMV2 switches running gateway.p4 - """ - super(self.__class__, self).__init__( - sw_info, 'TpsAggIngress', 'TpsEgress', proto_dump_file) - - def write_multicast_entry(self, hosts): - super(self.__class__, self).write_multicast_entry(hosts) - self.write_arp_flood() - - def add_data_inspection(self, dev_id, dev_mac): - logger.info( - 'Adding data inspection to aggregate device [%s] with device ID ' - '- [%s] and mac - [%s]', self.device_id, dev_id, dev_mac) - # Northbound Traffic Inspection for IPv4 - action_params = { - 'device': dev_id, - 'switch_id': self.int_device_id - } - table_entry = self.p4info_helper.build_table_entry( - table_name='{}.data_inspection_t'.format(self.p4_ingress), - match_fields={ - 'hdr.ethernet.src_mac': dev_mac, - }, - action_name='{}.data_inspect_packet'.format( - self.p4_ingress), - action_params=action_params - ) - self.write_table_entry(table_entry) - - logger.info( - 'Installed Northbound Packet Inspection for device - [%s]' - ' with MAC - [%s] with action params - [%s]', - AGG_CTRL_KEY, dev_mac, action_params) - - def del_data_inspection(self, dev_id, dev_mac): - logger.info( - 'Adding data inspection to aggregate device [%s] with device ID ' - '- [%s] and mac - [%s]', self.device_id, dev_id, dev_mac) - # Northbound Traffic Inspection for IPv4 - action_params = { - 'device': dev_id, - 'switch_id': self.int_device_id - } - table_entry = self.p4info_helper.build_table_entry( - table_name='{}.data_inspection_t'.format(self.p4_ingress), - match_fields={ - 'hdr.ethernet.src_mac': dev_mac, - }, - action_name='{}.data_inspect_packet'.format( - self.p4_ingress) - ) - self.delete_table_entry(table_entry) - - logger.info( - 'Installed Northbound Packet Inspection for device - [%s]' - ' with MAC - [%s] with action params - [%s]', - AGG_CTRL_KEY, dev_mac, action_params) - - def add_attack(self, **kwargs): - logger.info('Adding attack [%s]', kwargs) - action_name, dst_ipv4, dst_ipv6 = self.parse_attack(**kwargs) - self.insert_p4_table_entry( - table_name='data_drop_t', - action_name=action_name, - match_fields={ - 'hdr.ethernet.src_mac': kwargs['src_mac'], - 'meta.ipv4_addr': dst_ipv4, - 'meta.ipv6_addr': dst_ipv6, - 'meta.dst_port': int(kwargs['dst_port']), - }, - action_params=None, - ingress_class=True, - ) - logger.info('%s Dropping TCP Packets from %s', - self.name, kwargs.get('src_ip')) - - def stop_attack(self, **kwargs): - logger.info('Adding attack [%s]', kwargs) - action_name, dst_ipv4, dst_ipv6 = self.parse_attack(**kwargs) - - self.delete_p4_table_entry( - table_name='data_drop_t', - action_name=action_name, - match_fields={ - 'hdr.ethernet.src_mac': kwargs['src_mac'], - 'meta.ipv4_addr': dst_ipv4, - 'meta.ipv6_addr': dst_ipv6, - 'meta.dst_port': int(kwargs['dst_port']), - }, - ingress_class=True, - ) - logger.info('%s Dropping TCP Packets from %s', - self.name, kwargs.get('src_ip')) - - def add_switch_id(self): - action_params = { - 'switch_id': self.int_device_id - } - table_entry = self.p4info_helper.build_table_entry( - table_name='{}.add_switch_id_t'.format(self.p4_ingress), - match_fields={ - 'hdr.udp_int.dst_port': UDP_INT_DST_PORT - }, - action_name='{}.add_switch_id'.format( - self.p4_ingress), - action_params=action_params) - self.write_table_entry(table_entry) diff --git a/trans_sec/p4runtime_lib/core_switch.py b/trans_sec/p4runtime_lib/core_switch.py deleted file mode 100644 index 9f063e1f..00000000 --- a/trans_sec/p4runtime_lib/core_switch.py +++ /dev/null @@ -1,143 +0,0 @@ -# Copyright (c) 2020 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Originally copied from: -# -# Copyright 2017-present Open Networking Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import logging -import socket - -from trans_sec.consts import UDP_INT_DST_PORT -from trans_sec.p4runtime_lib.p4rt_switch import P4RuntimeSwitch - -logger = logging.getLogger('core_switch') - - -class CoreSwitch(P4RuntimeSwitch): - def __init__(self, sw_info, proto_dump_file=None): - """ - Construct Switch class to control BMV2 switches running gateway.p4 - """ - super(self.__class__, self).__init__( - sw_info, 'TpsCoreIngress', 'TpsCoreEgress', proto_dump_file) - - def write_multicast_entry(self, hosts): - try: - super(self.__class__, self).write_multicast_entry(hosts) - except Exception as e: - logger.warning('Unexpected error writing multicast entry - [%s]', - e) - self.write_arp_flood() - - def add_data_forward(self, dst_mac, egress_port): - logger.info( - 'Adding data forward to core device [%s] with source_mac ' - '- [%s] and ingress port - [%s]', - self.device_id, dst_mac, egress_port) - inserted = super(self.__class__, self).add_data_forward( - dst_mac, egress_port) - - if inserted: - table_entry = self.p4info_helper.build_table_entry( - table_name='{}.arp_forward_t'.format(self.p4_ingress), - match_fields={ - 'hdr.ethernet.dst_mac': dst_mac - }, - action_name='{}.arp_forward'.format(self.p4_ingress), - action_params={'port': egress_port} - ) - self.write_table_entry(table_entry) - - def add_data_inspection(self, dev_id, dev_mac): - logger.info( - 'Adding data inspection entry to core device [%s] with device ID ' - '- [%s]', self.int_device_id, dev_id) - - action_params = { - 'switch_id': self.int_device_id - } - table_name = '{}.data_inspection_t'.format(self.p4_ingress) - action_name = '{}.data_inspect_packet'.format(self.p4_ingress) - logger.info( - 'Insert params into table - [%s] for action [%s] ' - 'with params [%s]', - table_name, action_name, action_params) - table_entry = self.p4info_helper.build_table_entry( - table_name=table_name, - match_fields={ - 'hdr.udp_int.dst_port': UDP_INT_DST_PORT - }, - action_name=action_name, - action_params=action_params - ) - self.write_table_entry(table_entry) - - def del_data_inspection(self, dev_id, dev_mac): - logger.info( - 'Adding data inspection entry to core device [%s] with device ID ' - '- [%s]', self.device_id, dev_id) - - action_params = { - 'switch_id': dev_id - } - table_name = '{}.data_inspection_t'.format(self.p4_ingress) - action_name = '{}.data_inspect_packet'.format(self.p4_ingress) - logger.info( - 'Insert params into table - [%s] for action [%s] ' - 'with params [%s]', - table_name, action_name, action_params) - table_entry = self.p4info_helper.build_table_entry( - table_name=table_name, - match_fields={ - 'hdr.udp_int.dst_port': UDP_INT_DST_PORT - }, - action_name=action_name, - ) - self.delete_table_entry(table_entry) - - def setup_telemetry_rpt(self, ae_ip): - logger.info( - 'Setting up telemetry report on core device [%s] with ' - 'AE IP - [%s]', self.device_id, ae_ip) - - ae_ip_addr = socket.gethostbyname(ae_ip) - logger.info( - 'Starting telemetry report for INT headers with dst_port ' - 'value of 555 to AE IP [%s]', ae_ip_addr) - table_name = '{}.setup_telemetry_rpt_t'.format(self.p4_egress) - action_name = '{}.setup_telem_rpt_ipv4'.format(self.p4_egress) - match_fields = { - 'hdr.udp_int.dst_port': UDP_INT_DST_PORT - } - action_params = { - 'ae_ip': ae_ip_addr - } - table_entry = self.p4info_helper.build_table_entry( - table_name=table_name, - match_fields=match_fields, - action_name=action_name, - action_params=action_params) - self.write_table_entry(table_entry) diff --git a/trans_sec/p4runtime_lib/gateway_switch.py b/trans_sec/p4runtime_lib/gateway_switch.py deleted file mode 100644 index 064db846..00000000 --- a/trans_sec/p4runtime_lib/gateway_switch.py +++ /dev/null @@ -1,349 +0,0 @@ -# Copyright (c) 2020 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Originally copied from: -# -# Copyright 2017-present Open Networking Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import ipaddress -import logging -from abc import ABC - -from trans_sec.p4runtime_lib.p4rt_switch import P4RuntimeSwitch -from trans_sec.utils.convert import decode_num, decode_ipv4 - -logger = logging.getLogger('gateway_switch') - - -class GatewaySwitch(P4RuntimeSwitch, ABC): - def __init__(self, sw_info, proto_dump_file=None): - """ - Construct Switch class to control BMV2 switches running gateway.p4 - """ - super(self.__class__, self).__init__( - sw_info, 'TpsGwIngress', 'TpsEgress', proto_dump_file) - self.nat_udp_ports = set() - self.nat_tcp_ports = set() - self.tcp_port_count = 1 - self.udp_port_count = 1 - - def start_digest_listeners(self): - logger.debug('Starting digest listener for - [%s]', self.sw_info) - if 'arch' in self.sw_info and self.sw_info.get('arch') == 'tofino': - logger.info('Tofino currently not supporting digests') - pass - else: - logger.info('Building digest entry') - digest_entry, digest_info = self.p4info_helper.build_digest_entry( - digest_name="nat_digest") - self.write_digest_entry(digest_entry) - super(self.__class__, self).start_digest_listeners() - - def receive_nat_digests(self): - """ - Runnable method for self.digest_thread - """ - logger.info("Started listening NAT digest thread for %s", - self.name) - while True: - try: - logger.debug('Requesting NAT digests') - digests = self.digest_list() - digest_data = digests.digest.data - self.interpret_nat_digest(digest_data) - logger.debug('Interpreted NAT digest data') - except Exception as e: - logger.error( - 'Unexpected error reading NAT digest from [%s] - [%s]', - self.name, e) - - def add_data_inspection(self, dev_id, dev_mac): - logger.info( - 'Adding data inspection to gateway device [%s] with device ID ' - '- [%s] and mac - [%s]', self.device_id, dev_id, dev_mac) - # Northbound Traffic Inspection for IPv4 - action_params = { - 'device': dev_id, - 'switch_id': self.device_id - } - table_entry = self.p4info_helper.build_table_entry( - table_name='{}.data_inspection_t'.format(self.p4_ingress), - match_fields={ - 'hdr.ethernet.src_mac': dev_mac, - }, - action_name='{}.data_inspect_packet'.format( - self.p4_ingress), - action_params=action_params - ) - self.write_table_entry(table_entry) - - logger.info( - 'Installed Northbound Packet Inspection for device with' - ' MAC - [%s] with action params - [%s]', - dev_mac, action_params) - - def del_data_inspection(self, dev_id, dev_mac): - logger.info( - 'Adding data inspection to gateway device [%s] with device ID ' - '- [%s] and mac - [%s]', self.device_id, dev_id, dev_mac) - # Northbound Traffic Inspection for IPv4 - action_params = { - 'device': dev_id, - 'switch_id': self.device_id - } - table_entry = self.p4info_helper.build_table_entry( - table_name='{}.data_inspection_t'.format(self.p4_ingress), - match_fields={ - 'hdr.ethernet.src_mac': dev_mac, - }, - action_name='{}.data_inspect_packet'.format( - self.p4_ingress), - ) - self.delete_table_entry(table_entry) - - logger.info( - 'Installed Northbound Packet Inspection for device with' - ' MAC - [%s] with action params - [%s]', - dev_mac, action_params) - - @staticmethod - def parse_attack(**kwargs): - src_ip = ipaddress.ip_address(kwargs['src_ip']) - dst_ip = ipaddress.ip_address(kwargs['dst_ip']) - udp_table_name = 'data_drop_udp_ipv{}_t'.format(dst_ip.version) - tcp_table_name = 'data_drop_tcp_ipv{}_t'.format(dst_ip.version) - action_name = 'data_drop' - - logger.info('Attack src_ip - [%s], dst_ip - [%s]', src_ip, dst_ip) - # TODO - Add back source IP address as a match field after adding - # mitigation at the Aggregate - if dst_ip.version == 6: - logger.debug('Attack is IPv6') - proto_key = 'ipv6' - else: - logger.debug('Attack is IPv4') - proto_key = 'ipv4' - - dst_addr_key = 'hdr.{}.dstAddr'.format(proto_key) - - out = (udp_table_name, tcp_table_name, action_name, - str(dst_ip.exploded), dst_addr_key) - logger.info('Attack data - [%s]', out) - return out - - def add_attack(self, **kwargs): - logger.info('Adding attack [%s]', kwargs) - udp_tn, tcp_tn, action_name, dst_ip, dst_addr_key = \ - self.parse_attack(**kwargs) - - self.insert_p4_table_entry( - table_name=udp_tn, - action_name=action_name, - match_fields={ - 'hdr.ethernet.src_mac': kwargs['src_mac'], - dst_addr_key: str(dst_ip), - 'hdr.udp.dst_port': int(kwargs['dst_port']), - }, - action_params={'device': self.device_id}, - ingress_class=True, - ) - logger.info('%s Dropping UDP Packets from %s', - self.name, kwargs.get('src_ip')) - self.insert_p4_table_entry( - table_name=tcp_tn, - action_name=action_name, - match_fields={ - 'hdr.ethernet.src_mac': kwargs['src_mac'], - dst_addr_key: str(dst_ip), - 'hdr.tcp.dst_port': int(kwargs['dst_port']), - }, - action_params={'device': self.device_id}, - ingress_class=True, - ) - logger.info('%s Dropping TCP Packets from %s', - self.name, kwargs.get('src_ip')) - - def stop_attack(self, **kwargs): - logger.info('Stopping attack [%s]', kwargs) - udp_tn, tcp_tn, action_name, dst_ip, dst_addr_key = \ - self.parse_attack(**kwargs) - - self.delete_p4_table_entry( - table_name=udp_tn, - action_name=action_name, - match_fields={ - 'hdr.ethernet.src_mac': kwargs['src_mac'], - dst_addr_key: dst_ip, - 'hdr.udp.dst_port': int(kwargs['dst_port']), - }, - ingress_class=True, - ) - logger.info('%s no longer dropping UDP Packets from %s', - self.name, kwargs.get('src_ip')) - self.delete_p4_table_entry( - table_name=tcp_tn, - action_name=action_name, - match_fields={ - 'hdr.ethernet.src_mac': kwargs['src_mac'], - dst_addr_key: dst_ip, - 'hdr.tcp.dst_port': int(kwargs['dst_port']), - }, - ingress_class=True, - ) - logger.info('%s no longer dropping TCP Packets from %s', - self.name, kwargs.get('src_ip')) - - def add_nat_table(self, udp_source_port, tcp_source_port, source_ip): - gateway_public_ip = self.sw_info['public_ip'] - logger.info("Adding nat table entries on gateway device [%s] for %s", - self.device_id, source_ip) - logger.info("Check if %s not in %s for %s", udp_source_port, - self.nat_udp_ports, self.name) - # NAT Table Entries to handle UDP packets - if udp_source_port and udp_source_port not in self.nat_udp_ports: - table_entry = self.p4info_helper.build_table_entry( - table_name='{}.udp_local_to_global_t'.format(self.p4_ingress), - match_fields={ - 'hdr.udp.src_port': udp_source_port, - 'hdr.ipv4.srcAddr': (source_ip, 32) - }, - action_name='{}.udp_local_to_global'.format(self.p4_ingress), - action_params={ - 'src_port': int("50" + str(self.device_id) + str( - self.udp_port_count)), - 'ip_srcAddr': gateway_public_ip - }) - self.write_table_entry(table_entry) - table_entry = self.p4info_helper.build_table_entry( - table_name='{}.udp_global_to_local_t'.format(self.p4_ingress), - match_fields={ - 'hdr.udp.dst_port': int( - "50" + str(self.device_id) + str( - self.udp_port_count)), - 'hdr.ipv4.dstAddr': (gateway_public_ip, 32) - }, - action_name='{}.udp_global_to_local'.format(self.p4_ingress), - action_params={ - 'dst_port': udp_source_port, - 'ip_dstAddr': source_ip - }) - self.write_table_entry(table_entry) - self.udp_port_count = self.udp_port_count + 1 - self.nat_udp_ports.add(udp_source_port) - logger.info("UDP NAT table entry added on %s", - self.name) - elif tcp_source_port and tcp_source_port not in self.nat_tcp_ports: - # NAT Table Entries to handle TCP packets - table_entry = self.p4info_helper.build_table_entry( - table_name='{}.tcp_local_to_global_t'.format(self.p4_ingress), - match_fields={ - 'hdr.tcp.src_port': tcp_source_port, - 'hdr.ipv4.srcAddr': (source_ip, 32) - }, - action_name='{}.tcp_local_to_global'.format(self.p4_ingress), - action_params={ - 'src_port': int("50" + str(self.device_id) + str( - self.tcp_port_count)), - 'ip_srcAddr': gateway_public_ip - }) - self.write_table_entry(table_entry) - table_entry = self.p4info_helper.build_table_entry( - table_name='{}.tcp_global_to_local_t'.format(self.p4_ingress), - match_fields={ - 'hdr.tcp.dst_port': int( - "50" + str(self.device_id) + str( - self.tcp_port_count)), - 'hdr.ipv4.dstAddr': (gateway_public_ip, 32) - }, - action_name='{}.tcp_global_to_local'.format(self.p4_ingress), - action_params={ - 'dst_port': tcp_source_port, - 'ip_dstAddr': source_ip - }) - self.write_table_entry(table_entry) - self.tcp_port_count = self.tcp_port_count + 1 - self.nat_tcp_ports.add(tcp_source_port) - logger.info("TCP NAT table entry added on %s", - self.name) - - def interpret_nat_digest(self, digest_data): - logger.debug("Digest data %s", digest_data) - for members in digest_data: - logger.debug("Members: %s", members) - if members.WhichOneof('data') == 'struct': - udp_source_port = decode_num( - members.struct.members[0].bitstring) - logger.info('Learned UDP Source Port from %s is: %s', - self.name, udp_source_port) - tcp_source_port = decode_num( - members.struct.members[1].bitstring) - logger.info('Learned TCP Source Port from %s is: %s', - self.name, tcp_source_port) - source_ip = decode_ipv4(members.struct.members[2].bitstring) - logger.info('Learned Source IP Address from %s is: %s', - self.name, source_ip) - self.add_nat_table(udp_source_port, tcp_source_port, source_ip) - - def write_multicast_entry(self, hosts): - logger.debug('Writing multicast entries on gateway device [%s]', - self.device_id) - super(self.__class__, self).write_multicast_entry(hosts) - - # TODO/FIXME - We need to define something in the topology for - # determining which is the NB server as this will break outside of - # the tested topologies and probably has other unintended consequences - target_host = None - - logger.debug('Switch Hosts defs - [%s]', hosts) - - for host in hosts.values(): - logger.debug('Switch Host def - [%s]', host) - if host['type'] == 'target-server': - target_host = host - - if target_host: - table_entry = self.p4info_helper.build_table_entry( - table_name='{}.mac_lookup_ipv4_t'.format(self.p4_ingress), - match_fields={ - 'hdr.ipv4.dstAddr': (target_host['ip'], 32) - }, - action_name='{}.mac_lookup'.format(self.p4_ingress), - action_params={ - 'dst_mac': target_host['mac'] - }) - self.write_table_entry(table_entry) - table_entry = self.p4info_helper.build_table_entry( - table_name='{}.mac_lookup_ipv6_t'.format(self.p4_ingress), - match_fields={ - 'hdr.ipv6.dstAddr': (target_host['ipv6'], 128) - }, - action_name='{}.mac_lookup'.format(self.p4_ingress), - action_params={ - 'dst_mac': target_host['mac'] - }) - self.write_table_entry(table_entry) - else: - logger.warning('Target host not found, not setting the ' - 'multicast group') diff --git a/trans_sec/p4runtime_lib/helper.py b/trans_sec/p4runtime_lib/helper.py deleted file mode 100644 index f0fe12af..00000000 --- a/trans_sec/p4runtime_lib/helper.py +++ /dev/null @@ -1,297 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Originally copied from: -# -# Copyright 2017-present Open Networking Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import logging -import re - -import google.protobuf.text_format -from p4.config.v1 import p4info_pb2 -from p4.v1 import p4runtime_pb2 - -from trans_sec.utils.convert import encode - -logger = logging.getLogger('helper') - - -class P4InfoHelper(object): - def __init__(self, p4_info_filepath): - p4info = p4info_pb2.P4Info() - # Load the p4info file into a skeleton P4Info object - with open(p4_info_filepath) as p4info_f: - google.protobuf.text_format.Merge(p4info_f.read(), p4info) - self.p4info = p4info - - def get(self, entity_type, name=None, entity_id=None): - if name is not None and entity_id is not None: - raise AssertionError("name or id must be None") - - for o in getattr(self.p4info, entity_type): - pre = o.preamble - if name: - if pre.name == name or pre.alias == name: - return o - else: - if pre.id == entity_id: - return o - - if name: - raise AttributeError( - "Could not find %r of type %s" % (name, entity_type)) - else: - raise AttributeError( - "Could not find id %r of type %s" % (entity_id, entity_type)) - - def get_id(self, entity_type, name): - return self.get(entity_type, name=name).preamble.id - - def get_name(self, entity_type, entity_id): - return self.get(entity_type, entity_id=entity_id).preamble.name - - def get_alias(self, entity_type, entity_id): - return self.get(entity_type, entity_id=entity_id).preamble.alias - - def __getattr__(self, attr): - # Synthesize convenience functions for name to id lookups for top-level - # entities - # e.g. get_tables_id(name_string) or get_actions_id(name_string) - m = re.search(r"^get_(\w+)_id$", attr) - if m: - primitive = m.group(1) - return lambda name: self.get_id(primitive, name) - - # Synthesize convenience functions for id to name lookups - # e.g. get_tables_name(id) or get_actions_name(id) - m = re.search(r"^get_(\w+)_name$", attr) - if m: - primitive = m.group(1) - return lambda prim_id: self.get_name(primitive, prim_id) - - raise AttributeError( - "%r object has no attribute %r" % (self.__class__, attr)) - - def get_match_field(self, table_name, name=None, field_id=None): - for t in self.p4info.tables: - pre = t.preamble - if pre.name == table_name: - for mf in t.match_fields: - if name is not None: - if mf.name == name: - return mf - elif field_id is not None: - if mf.id == field_id: - return mf - raise AttributeError("%r has no attribute %r" % ( - table_name, name if name is not None else field_id)) - - def get_match_field_id(self, table_name, match_field_name): - return self.get_match_field(table_name, name=match_field_name).id - - def get_match_field_name(self, table_name, match_field_id): - return self.get_match_field(table_name, field_id=match_field_id).name - - def get_match_field_pb(self, table_name, match_field_name, value): - logger.info( - 'Retrieving match field on table name - [%s], match field - [%s], ' - 'value - [%s]', table_name, match_field_name, value) - - p4info_match = self.get_match_field(table_name, match_field_name) - bit_width = p4info_match.bitwidth - p4runtime_match = p4runtime_pb2.FieldMatch() - p4runtime_match.field_id = p4info_match.id - match_type = p4info_match.match_type - - logger.info( - 'Encoding value [%s] for exact match with bitwidth - [%s]', - value, bit_width) - - if match_type == p4info_pb2.MatchField.EXACT: - logger.info('Encoding for EXACT matches') - exact = p4runtime_match.exact - if isinstance(value, list) or isinstance(value, tuple): - exact.value = encode(value[0], bit_width) - else: - exact.value = encode(value, bit_width) - elif match_type == p4info_pb2.MatchField.LPM: - logger.info('Encoding for LPM matches') - lpm = p4runtime_match.lpm - lpm.value = encode(value[0], bit_width) - lpm.prefix_len = value[1] - elif match_type == p4info_pb2.MatchField.TERNARY: - logger.info('Encoding for TERNARY matches') - ternary = p4runtime_match.ternary - ternary.value = encode(value[0], bit_width) - ternary.mask = encode(value[1], bit_width) - elif match_type == p4info_pb2.MatchField.RANGE: - logger.info('Encoding for RANGE matches') - range_match = p4runtime_match.range - range_match.low = encode(value[0], bit_width) - range_match.high = encode(value[1], bit_width) - else: - raise Exception("Unsupported match type with type %r" % match_type) - return p4runtime_match - - @staticmethod - def get_match_field_value(match_field): - match_type = match_field.WhichOneof("field_match_type") - if match_type == 'valid': - return match_field.valid.value - elif match_type == 'exact': - return match_field.exact.value - elif match_type == 'lpm': - return match_field.lpm.value, match_field.lpm.prefix_len - elif match_type == 'ternary': - return match_field.ternary.value, match_field.ternary.mask - elif match_type == 'range': - return match_field.range.low, match_field.range.high - else: - raise Exception("Unsupported match type with type %r" % match_type) - - def get_action_param(self, action_name, name=None, action_id=None): - action = None - for action in self.p4info.actions: - pre = action.preamble - if pre.name == action_name: - for param in action.params: - if name is not None: - if param.name == name: - return param - elif action_id is not None: - if param.id == action_id: - return param - if action: - raise AttributeError("action %r has no param %r, (has: %r)" % ( - action_name, name if name is not None else action_id, - action.params)) - else: - raise AttributeError("action %r has no param %r" % ( - action_name, name if name is not None else action_id)) - - def get_action_param_id(self, action_name, param_name): - return self.get_action_param(action_name, name=param_name).id - - def get_action_param_name(self, action_name, param_id): - return self.get_action_param(action_name, action_id=param_id).name - - def get_action_param_pb(self, action_name, param_name, value): - logger.info( - 'Retrieving action param for action - [%s], param - [%s], ' - 'value - [%s]', action_name, param_name, value) - p4info_param = self.get_action_param(action_name, param_name) - p4runtime_param = p4runtime_pb2.Action.Param() - p4runtime_param.param_id = p4info_param.id - p4runtime_param.value = encode(value, p4info_param.bitwidth) - return p4runtime_param - - def build_table_entry(self, table_name, match_fields=None, - default_action=False, action_name=None, - action_params=None, priority=None): - logger.info( - 'Building table entry to table [%s] with match_fields - [%s] ' - 'action - [%s] and params - [%s]', - table_name, match_fields, action_name, action_params) - table_entry = p4runtime_pb2.TableEntry() - table_entry.table_id = self.get_tables_id(table_name) - logger.debug('Table ID - [%s]', table_entry.table_id) - - if priority is not None: - table_entry.priority = priority - - if match_fields: - table_entry.match.extend([ - self.get_match_field_pb(table_name, match_field_name, value) - for match_field_name, value in match_fields.items() - ]) - - if default_action: - table_entry.is_default_action = True - - if action_name: - action = table_entry.action.action - action.action_id = self.get_actions_id(action_name) - if action_params: - logger.info('Action params - [%s]', action_params) - action.params.extend([ - self.get_action_param_pb(action_name, field_name, value) - for field_name, value in action_params.items() - ]) - return table_entry - - @staticmethod - def reset_counter(counter_id, index): - logger.info('Resetting counter with ID - [%s] and index - [%s]', - counter_id, index) - counter_entry = p4runtime_pb2.CounterEntry() - if counter_id is not None: - counter_entry.counter_id = counter_id - else: - counter_entry.counter_id = 0 - if index is not None: - counter_entry.index.index = index - counter_entry.data.byte_count = 0 - counter_entry.data.packet_count = 0 - - @staticmethod - def build_clone_entry(clone_egress): - logger.info('Building clone entry with egress_port value of - [%s]', - clone_egress) - pre_entry = p4runtime_pb2.PacketReplicationEngineEntry() - clone_session_entry = p4runtime_pb2.CloneSessionEntry() - clone_session_entry.session_id = 5 - clone_session_entry.replicas.add(egress_port=clone_egress, instance=1) - pre_entry.clone_session_entry.CopyFrom(clone_session_entry) - logger.info('Clone entry - [%s]', pre_entry) - return pre_entry - - def build_digest_entry(self, digest_name): - digest_info = {} - logger.info("Building digest entry for %s", digest_name) - digest_entry = p4runtime_pb2.DigestEntry() - # using name - # TODO/FIXME - This appears to be broken on Tofino see issue #175 - logger.debug('Before getting digest ID') - digest_entry.digest_id = self.get_digests_id(digest_name) - logger.debug('After getting digest ID') - # using id directly - digest_entry.config.max_timeout_ns = 0 - digest_entry.config.max_list_size = 1 - digest_entry.config.ack_timeout_ns = 0 - digest_info[digest_name] = digest_entry.digest_id - logger.info("Digest Entry Information %s", digest_info) - return digest_entry, digest_info - - @staticmethod - def build_multicast_group_entry(mc_group_id, replicas): - mc_entry = p4runtime_pb2.PacketReplicationEngineEntry() - mc_entry.multicast_group_entry.multicast_group_id = mc_group_id - for replica in replicas: - r = p4runtime_pb2.Replica() - r.egress_port = int(replica['egress_port']) - r.instance = int(replica['instance']) - mc_entry.multicast_group_entry.replicas.extend([r]) - return mc_entry diff --git a/trans_sec/p4runtime_lib/p4rt_switch.py b/trans_sec/p4runtime_lib/p4rt_switch.py deleted file mode 100644 index dea1a3ef..00000000 --- a/trans_sec/p4runtime_lib/p4rt_switch.py +++ /dev/null @@ -1,624 +0,0 @@ -# Copyright (c) 2020 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import codecs -import logging -import struct -from abc import ABC -from threading import Thread - -import grpc -from p4.tmp import p4config_pb2 -from p4.v1 import p4runtime_pb2, p4runtime_pb2_grpc - -from trans_sec.controller.ddos_sdn_controller import GATEWAY_CTRL_KEY -from trans_sec.p4runtime_lib import helper -from trans_sec.switch import SwitchConnection, IterableQueue, GrpcRequestLogger -from trans_sec.utils.convert import decode_mac - -MSG_LOG_MAX_LEN = 1024 - -logger = logging.getLogger('switch') - - -class P4RuntimeSwitch(SwitchConnection, ABC): - - def __init__(self, sw_info, p4_ingress, p4_egress, proto_dump_file=None): - super(P4RuntimeSwitch, self).__init__(sw_info) - p4info_txt = sw_info['runtime_p4info'] - logger.info('Loading p4info_helper with file - [%s]', - p4info_txt) - self.p4info_helper = helper.P4InfoHelper(p4info_txt) - self.p4_ingress = p4_ingress - self.p4_egress = p4_egress - grpc_channel = grpc.insecure_channel(self.grpc_addr) - if proto_dump_file is not None: - logger.info('Adding interceptor with file - [%s] to device [%s]', - proto_dump_file, self.grpc_addr) - interceptor = GrpcRequestLogger(proto_dump_file) - grpc_channel = grpc.intercept_channel(grpc_channel, interceptor) - - logger.info('Creating client stub to channel - [%s] at address - [%s]', - grpc_channel, self.grpc_addr) - self.client_stub = p4runtime_pb2_grpc.P4RuntimeStub(grpc_channel) - self.requests_stream = IterableQueue() - self.stream_msg_resp = self.client_stub.StreamChannel( - iter(self.requests_stream)) - self.proto_dump_file = proto_dump_file - self.digest_thread = Thread(target=self.receive_digests) - - def start_digest_listeners(self): - logger.info( - 'Starting mac_learn_digest Digest for device [%s] named [%s]', - self.grpc_addr, self.name) - digest_entry, digest_info = self.p4info_helper.build_digest_entry( - digest_name="mac_learn_digest") - self.write_digest_entry(digest_entry) - - logger.info('Starting digest threads') - self.digest_thread.start() - - def stop_digest_listeners(self): - if 'arch' in self.sw_info and self.sw_info['arch'] == 'tofino': - logger.info('Tofino currently not supporting digests') - pass - else: - self.requests_stream.close() - self.stream_msg_resp.cancel() - self.digest_thread.join() - - def receive_digests(self): - """ - Runnable method for self.digest_thread - """ - logger.info("Started listening digest thread on device [%s] with " - "name [%s]", self.grpc_addr, self.name) - while True: - try: - logger.debug('Requesting digests from device [%s]', - self.grpc_addr) - digests = self.digest_list() - logger.debug('digests from device [%s] - [%s]', - self.grpc_addr, digests) - digest_data = digests.digest.data - logger.debug('Received digest data from device [%s]: [%s]', - self.grpc_addr, digest_data) - self.interpret_digest(digest_data) - logger.debug('Interpreted digest data') - except Exception as e: - logger.error( - 'Unexpected error reading digest from device [%s] - [%s]', - self.grpc_addr, e) - - def interpret_digest(self, digest_data): - logger.debug("Digest data from switch [%s] - [%s]", - self.name, digest_data) - - if not digest_data or len(digest_data) == 0: - logger.warning('No digest data to process') - return - for members in digest_data: - logger.debug("Digest members: %s", members) - if members.WhichOneof('data') == 'struct': - source_mac = decode_mac(members.struct.members[0].bitstring) - if source_mac: - logger.debug('Digest MAC Address is: %s', source_mac) - ingress_port = int( - codecs.encode(members.struct.members[1].bitstring, - 'hex'), 16) - logger.debug('Digest Ingress Port is %s', ingress_port) - self.add_data_forward(source_mac, ingress_port) - else: - logger.warning('Could not retrieve source_mac from digest') - else: - logger.warning('Digest could not be processed - [%s]', - digest_data) - - logger.info('Completed digest processing') - - def get_table_entry(self, table_name, action_name, match_fields, - action_params, ingress_class=True): - logger.info("Access P4 table [%s]", table_name) - if ingress_class: - tbl_class = self.p4_ingress - else: - tbl_class = self.p4_egress - - return self.p4info_helper.build_table_entry( - table_name='{}.{}'.format(tbl_class, table_name), - match_fields=match_fields, - action_name='{}.{}'.format(tbl_class, action_name), - action_params=action_params) - - def insert_p4_table_entry(self, table_name, action_name, match_fields, - action_params, ingress_class=True, - election_high=0, election_low=1): - - table_entry = self.get_table_entry( - table_name, action_name, match_fields, action_params, - ingress_class) - logger.debug( - 'Writing table entry to device [%s] table [%s], ' - 'with action name - [%s], ' - 'match fields - [%s], action_params - [%s]', - self.grpc_addr, table_name, action_name, match_fields, - action_params) - self.write_table_entry(table_entry=table_entry, - election_high=election_high, - election_low=election_low) - - def delete_p4_table_entry(self, table_name, action_name, match_fields, - action_params=None, ingress_class=True, - election_high=0, election_low=1): - - table_entry = self.get_table_entry( - table_name, action_name, match_fields, action_params, - ingress_class) - logger.debug( - 'Deleting table entry to device [%s] table [%s], ' - 'with action name - [%s], ' - 'match fields - [%s], action_params - [%s]', - self.grpc_addr, table_name, action_name, match_fields, - action_params) - self.delete_table_entry(table_entry, election_high, election_low) - - def __get_data_forward_table_entry(self, dst_mac, egress_port): - logger.info( - 'Adding data forward to device [%s] with destination MAC ' - '- [%s] and egress port - [%s]', - self.grpc_addr, dst_mac, egress_port) - - if self.sw_info['type'] == GATEWAY_CTRL_KEY: - action_params = { - 'port': egress_port, - 'switch_mac': self.mac - } - else: - action_params = {'port': egress_port} - - table_name = '{}.data_forward_t'.format(self.p4_ingress) - - table_keys = self.get_data_forward_macs() - logger.debug('Table keys to [%s] - [%s] on device [%s]', - table_name, table_keys, self.grpc_addr) - - if dst_mac not in table_keys: - logger.info('Returning table entry for data_forward_t insertion') - return self.p4info_helper.build_table_entry( - table_name=table_name, - match_fields={ - 'hdr.ethernet.dst_mac': dst_mac - }, - action_name='{}.data_forward'.format(self.p4_ingress), - action_params=action_params - ) - elif not egress_port: - logger.info('Returning table entry for data_forward_t deletion') - return self.p4info_helper.build_table_entry( - table_name=table_name, - match_fields={ - 'hdr.ethernet.dst_mac': dst_mac - }, - action_name='{}.data_forward'.format(self.p4_ingress), - ) - else: - logger.info( - 'Data forward entry already inserted with key - [%s] ' - 'on device [%s]', - dst_mac, self.grpc_addr) - return None - - def add_data_forward(self, dst_mac, egress_port): - table_entry = self.__get_data_forward_table_entry(dst_mac, egress_port) - if table_entry: - self.write_table_entry(table_entry) - return True - else: - logger.warning( - 'Unable to insert data_forward request with mac - [%s] & ' - 'port - [%s]', dst_mac, egress_port) - return False - - def del_data_forward(self, dst_mac): - table_entry = self.__get_data_forward_table_entry(dst_mac, None) - if table_entry: - self.delete_table_entry(table_entry) - return True - else: - logger.warning( - 'Unable to delete data_forward request with mac - [%s]', - dst_mac) - return False - - def add_data_inspection(self, dev_id, dev_mac): - logger.info( - 'Adding data inspection to device [%s] with ID ' - '- [%s] and mac - [%s]', - self.device_id, self.int_device_id, dev_mac) - action_params = { - 'device': self.int_device_id, - 'switch_id': self.sw_info['id'] - } - table_entry = self.p4info_helper.build_table_entry( - table_name='{}.data_inspection_t'.format(self.p4_ingress), - match_fields={ - 'hdr.ethernet.src_mac': dev_mac, - }, - action_name='{}.data_inspect_packet'.format( - self.p4_ingress), - action_params=action_params - ) - self.write_table_entry(table_entry) - - logger.info( - 'Installed Northbound Packet Inspection for device with' - ' MAC - [%s] with action params - [%s]', - dev_mac, action_params) - - def del_data_inspection(self, dev_id, dev_mac): - logger.info( - 'Removing data inspection to device [%s] with ID ' - '- [%s] and mac - [%s]', self.device_id, dev_id, dev_mac) - table_entry = self.p4info_helper.build_table_entry( - table_name='{}.data_inspection_t'.format(self.p4_ingress), - match_fields={ - 'hdr.ethernet.src_mac': dev_mac, - }, - action_name='{}.data_inspect_packet'.format( - self.p4_ingress), - ) - self.delete_table_entry(table_entry) - - logger.info( - 'Installed Northbound Packet Inspection for device with' - ' MAC - [%s]', dev_mac) - - def add_switch_id(self): - pass - - def build_device_config(self): - if self.sw_info['type'] == 'tofino': - return self.__build_device_config_tofino() - else: - return self.__build_device_config_bmv2() - - def __build_device_config_bmv2(self): - runtime_json = self.sw_info['runtime_json'] - logger.info('Building device [%s] config with file - [%s]', - self.grpc_addr, runtime_json) - - device_config = p4config_pb2.P4DeviceConfig() - device_config.reassign = True - with open(runtime_json) as f: - file_data = f.read() - device_config.device_data = bytes(file_data, 'utf-8') - return device_config - - def __build_device_config_tofino(self): - """ - Builds the device config for Tofino - """ - prog_name = self.sw_info['type'] - bin_path = self.sw_info['bin_path'] - cxt_json_path = self.sw_info['cxt_json_path'] - logger.info( - 'Building device configuration for program - [%s], bin_path - [%s]' - ', and cxt_json_path - [%s]', prog_name, bin_path, cxt_json_path) - prog_name = prog_name.encode('utf-8') - device_config = p4config_pb2.P4DeviceConfig() - device_config.reassign = True - with open(bin_path, 'rb') as bin_f: - with open(cxt_json_path, 'r') as cxt_json_f: - device_config.device_data = "" - device_config.device_data += struct.pack("' + logfile + ' 2>&1 & echo $! >> ' + f.name) - pid = int(f.read()) - logger.debug("P4 switch {} PID is {}.".format(self.name, pid)) - if not self.check_switch_started(pid): - raise Exception("P4 switch {} did not start correctly.\n".format( - self.name)) - logger.info("P4 switch %s has been started.", self.name) From 36a546cdbb46da0886ff411dc97f8257fe1a3d07 Mon Sep 17 00:00:00 2001 From: spisarski Date: Fri, 6 Aug 2021 10:49:46 -0600 Subject: [PATCH 15/40] Few more files missed after the first pass --- README.md | 1 - bin/bfrt_connect.py | 100 ------------ bin/bfrt_switch_tester.py | 99 ------------ bin/bfrt_test.py | 97 ------------ bin/call_sdn_api.py | 84 ---------- bin/full-gen-pcap-frg.sh | 26 ---- bin/send.py | 130 ---------------- bin/write_to_clone.py | 75 --------- bin/write_to_p4_table.py | 146 ------------------ .../test_cases/write_clone_entry.yml | 40 ----- .../test_cases/write_to_p4_table.yml | 68 -------- trans_sec/bfruntime_lib/gateway_switch.py | 53 ------- 12 files changed, 919 deletions(-) delete mode 100755 bin/bfrt_connect.py delete mode 100644 bin/bfrt_switch_tester.py delete mode 100644 bin/bfrt_test.py delete mode 100755 bin/call_sdn_api.py delete mode 100644 bin/full-gen-pcap-frg.sh delete mode 100755 bin/send.py delete mode 100755 bin/write_to_clone.py delete mode 100755 bin/write_to_p4_table.py delete mode 100644 playbooks/scenarios/test_cases/write_clone_entry.yml delete mode 100644 playbooks/scenarios/test_cases/write_to_p4_table.yml delete mode 100644 trans_sec/bfruntime_lib/gateway_switch.py diff --git a/README.md b/README.md index abfeb047..0b7fc870 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,5 @@ Questions? Just send us an email at - docs - miscellaneous MD files - p4 - The P4 source code - playbooks - The Ansible Playbooks used by automation -- snaps-hcp - Documentation on snaps-hcp which is no longer being used - tests - the Python unit test directory - trans_sec - the project's top-level Python package diff --git a/bin/bfrt_connect.py b/bin/bfrt_connect.py deleted file mode 100755 index 68f68651..00000000 --- a/bin/bfrt_connect.py +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2020 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import argparse -import logging -import sys - -import bfrt_grpc.client as bfrt_client - - -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument('-g', '--grpc-addr', required=False, - default='localhost:50052', - help='The GRPC address in the form host:port') - parser.add_argument('-n', '--program-name', required=False, - default=None, - help='The P4 program name') - parser.add_argument('-c', '--client-id', required=False, default=0, - type=int, help='The client ID - default 0') - parser.add_argument('-d', '--device-id', required=False, default=0, - type=int, help='The device ID - default 0') - parser.add_argument('-m', '--is-master', required=False, type=bool, - default=True, help='Is master client') - parser.add_argument('-t', '--table-name', required=False, - help='The table name for logging the ID') - parser.add_argument('-l', '--log-dir', type=str, required=False, - default=None) - parser.add_argument('-lf', '--log-file', type=str, required=False, - default='insert_p4_table.log') - parser.add_argument('-R', '--reset', '--clear', type=bool, required=False, - default=False, - help='Clear all tables before programming') - return parser.parse_args() - - -if __name__ == '__main__': - """ - Inserts entries into P4 tables based on the entries within table config - file - """ - args = get_args() - logger = logging.getLogger('bfrt_connect') - - # - # Set up Logging - # - if args.log_dir and args.log_file: - log_file = '{}/{}'.format(args.log_dir, args.log_file) - logging.basicConfig(level=logging.DEBUG, filename=log_file) - else: - logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) - - # - # Connect to the BF Runtime Server - # - logger.info('Creating client interface with client stub') - logger.debug('grpc-addr - [%s], client_id - [%s], device_id - [%s], ' - 'is_master - [%s]', - args.grpc_addr, args.client_id, args.device_id, - args.is_master) - - interface = bfrt_client.ClientInterface( - grpc_addr=args.grpc_addr, client_id=args.client_id, - device_id=args.device_id, is_master=args.is_master) - - # - # Get the information about the running program - # - bfrt_info = interface.bfrt_info_get(args.program_name) - - # Attempt to receive the ID for the args.table_name argument - if args.table_name: - logger.info('Retrieve table name - [%s]', args.table_name) - try: - table = bfrt_info.table_get(args.table_name) - logger.info('Table ID - [%s]', table.info.id_get()) - except Exception as e: - logger.error("Exit with error - [%s]", e) - - # - # A small workaround to close the connection properly. That should be - # addressed in the future versions of SDE - interface._tear_down_stream() - - # - # The End - # - logger.info('Exit 0') diff --git a/bin/bfrt_switch_tester.py b/bin/bfrt_switch_tester.py deleted file mode 100644 index 511f2b4b..00000000 --- a/bin/bfrt_switch_tester.py +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2020 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import argparse -import json -import logging -import sys - -import yaml - -from trans_sec.bfruntime_lib.aggregate_switch import AggregateSwitch -from trans_sec.bfruntime_lib.core_switch import CoreSwitch -from trans_sec.bfruntime_lib.gateway_switch import GatewaySwitch - - -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument('-t', '--topo', required=True, - help='The location of the topology file') - parser.add_argument('-n', '--program-name', required=False, - default=None, - help='The P4 program name') - parser.add_argument('-p', '--proto-dump-file', required=False, - default=None, help='The GRPC logger') - parser.add_argument('-c', '--client-id', required=False, default=0, - type=int, help='The client ID - default 0') - parser.add_argument('-l', '--log-dir', type=str, required=False, - default=None) - parser.add_argument('-lf', '--log-file', type=str, required=False, - default='insert_p4_table.log') - return parser.parse_args() - - -if __name__ == '__main__': - """ - Inserts entries into P4 tables based on the entries within table config - file - """ - args = get_args() - # - # Set up Logging - # - if args.log_dir and args.log_file: - log_file = '{}/{}'.format(args.log_dir, args.log_file) - logging.basicConfig(level=logging.DEBUG, filename=log_file) - else: - logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) - - logger = logging.getLogger('bfrt_switch_tester') - - topo_file = args.topo - with open(topo_file, 'r') as f: - if topo_file.endswith('json'): - topo = json.load(f) - else: - topo = yaml.safe_load(f) - - logger.info('topo - [%s]', topo) - - for sw_info in topo['switches'].values(): - logger.info('sw_info - [%s]', sw_info) - sw_type = sw_info['type'] - switch = None - if sw_type == 'core': - logger.info('Instantiating CoreSwitch') - switch = CoreSwitch(sw_info) - elif sw_type == 'aggregate': - logger.info('Instantiating AggregateSwitch') - switch = AggregateSwitch(sw_info) - elif sw_type == 'gateway': - logger.info('Instantiating GatewaySwitch') - switch = GatewaySwitch(sw_info, args.proto_dump_file) - - logger.info('Adding switch ID to switch - [%s]', sw_info['id']) - - if switch: - switch.add_switch_id(sw_info['id']) - - logger.info('Starting digest listeners') - switch.start_digest_listeners() - - logger.info('Stopping switch') - switch.stop_digest_listeners() - - # - # The End - # - logger.info('Exit 0') diff --git a/bin/bfrt_test.py b/bin/bfrt_test.py deleted file mode 100644 index db2eb144..00000000 --- a/bin/bfrt_test.py +++ /dev/null @@ -1,97 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2020 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import print_function - -import logging -import os -import sys - -import argparse -import bfrt_grpc.client as bfrt_client - -SDE_INSTALL = os.environ['SDE_INSTALL'] -SDE_PYTHON_27 = os.path.join(SDE_INSTALL, 'lib', 'python2.7', 'site-packages') - -sys.path.append(SDE_PYTHON_27) -sys.path.append(os.path.join(SDE_PYTHON_27, 'tofino')) - - -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument('-g', '--grpc-addr', required=True, - help='The GRPC address in the form host:port') - parser.add_argument('-i', '--ingress', required=True, type=str, - help='The name of the table to manipulate') - parser.add_argument('-d', '--device_id', required=False, type=int, - default=0, help='The name of the table to manipulate') - parser.add_argument('-t', '--table', required=False, type=str, - default='data_forward_t', - help='The name of the table to manipulate') - parser.add_argument('-a', '--action', required=False, type=str, - default='data_forward', - help='The name of the table to manipulate') - parser.add_argument('-f', '--mac-field', required=False, type=str, - default='hdr.ethernet.dst_mac', - help='The name of the table to manipulate') - parser.add_argument('-m', '--mac', required=False, type=str, - default='00:00:00:01:01:01', - help='The key value to the data_forward_t table') - parser.add_argument('-p', '--port', required=False, type=int, - default=2, - help='The acton value to the data_forward action') - return parser.parse_args() - - -if __name__ == '__main__': - """ - Inserts entries into P4 tables based on the entries within table config - file - """ - args = get_args() - logger = logging.getLogger('bfrt_connect') - logging.basicConfig(level=logging.DEBUG) - # - # Connect to the BF Runtime Server - # - interface = bfrt_client.ClientInterface( - grpc_addr=args.grpc_addr, - client_id=0, - device_id=0, - is_master=True) - logger.info('Connected to BF Runtime Server') - - target = bfrt_client.Target(device_id=args.device_id, pipe_id=0xffff) - # - # Get the information about the running program - # - bfrt_info = interface.bfrt_info_get() - p4_name = bfrt_info.p4_name_get() - logger.info('The target runs program ', p4_name) - interface.bind_pipeline_config(p4_name) - - table_name = "{}.{}".format(args.ingress, args.table) - logger.info('Table name - [%s]', table_name) - table = bfrt_info.table_get(table_name) - logger.info('Table class - [%s]', table.__class__) - table.info.key_field_annotation_add(args.mac_field, 'mac') - - action_name = "{}.{}".format(args.ingress, args.action) - - table.entry_add(target, - [bfrt_client.KeyTuple(args.mac_field, args.mac)], - [bfrt_client.DataTuple('port', val=args.port)]) - - interface._tear_down_stream() diff --git a/bin/call_sdn_api.py b/bin/call_sdn_api.py deleted file mode 100755 index f34d25fb..00000000 --- a/bin/call_sdn_api.py +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import argparse -import json -import logging - -import pydevd -import requests - -logger = logging.getLogger('oinc') -FORMAT = '%(levelname)s %(asctime)-15s %(filename)s %(lineno)d %(message)s' - - -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument( - '-l', '--loglevel', - help='Log Level defaults to INFO', - required=False, default='INFO') - parser.add_argument( - '-s', '--sdn-url', dest='sdn_url', required=False, - default='localhost:9998', help='the URL to the SDN controller') - parser.add_argument( - '-o', '--operation', dest='operation', required=False, - help='the operation to call', default='attack') - parser.add_argument( - '-p', '--protocol', dest='protocol', required=False, - help='the protocol to call', default='http') - - parser.add_argument( - '-dh', '--debug-host', dest='debug_host', - help='remote debugging host IP') - parser.add_argument( - '-dp', '--debug-port', dest='debug_port', default=5678, - help='the remote debugging port') - - attack = { - 'src_mac': '00:00:00:00:03:01', - 'src_ip': '10.0.1.6', - 'dst_ip': '10.0.1.8', - 'dst_port': '4323', - 'packet_size': '86', - 'attack_type': 'UDP Flood', - } - parser.add_argument('-a', '--rest-args', dest='rest_args', required=False, - default=attack, type=json.loads, - help='the REST call dict arguments') - return parser.parse_args() - - -def main(): - args = get_args() - - # Initialize logger - numeric_level = getattr(logging, args.loglevel.upper(), None) - logging.basicConfig(format=FORMAT, level=numeric_level) - - # Setup remote debugging - if args.debug_host: - pydevd.settrace(host=args.debug_host, port=int(args.debug_port), - stdoutToServer=True, stderrToServer=True, - suspend=False) - logger.info('Starting Oinc with SDN Controller url [%s]', args.sdn_url) - - logger.info('Retrieving http session from url - [%s]', args.sdn_url) - url = '{}://{}/{}'.format(args.protocol, args.sdn_url, args.operation) - ret_val = requests.post(url=url, params=args.rest_args) - logger.info('Return value of REST call [%s]', ret_val) - - -if __name__ == '__main__': - main() diff --git a/bin/full-gen-pcap-frg.sh b/bin/full-gen-pcap-frg.sh deleted file mode 100644 index 28bc31c0..00000000 --- a/bin/full-gen-pcap-frg.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash - -# Contains the packet generation and sniffing commands for generating INT and non-INT PCAP files - -# With INT -sudo python /home/ubuntu/transparent-security/trans_sec/device_software/send_packets.py -y 0 -i 0.005 -it 1 -itd 0 -z gateway1-eth1 -sa 192.168.1.2 -r 10.0.1.8 -sp 9648 -p 3074 -c 1 -e 00:00:00:00:01:01 -s 00:00:00:01:01:00 -pr UDP -m 'UDP INT IPv4' -sudo python /home/ubuntu/transparent-security/trans_sec/device_software/send_packets.py -y 0 -i 0.005 -it 1 -itd 0 -z gateway1-eth1 -sa 192.168.1.2 -r 10.0.1.8 -sp 9648 -p 3074 -c 1 -e 00:00:00:00:01:01 -s 00:00:00:01:01:00 -pr TCP -m 'TCP INT IPv4' - -sudo python /home/ubuntu/transparent-security/trans_sec/device_software/send_packets.py -y 0 -i 0.005 -it 1 -itd 0 -z gateway1-eth1 -sa 0000:0000:0000:0000:0000:0001:0001:0002 -r 0000:0000:0000:0000:0000:0002:0001:0002 -sp 9648 -p 3074 -c 1 -e 00:00:00:00:01:01 -s 00:00:00:01:01:00 -pr UDP -m 'UDP INT IPv6' -sudo python /home/ubuntu/transparent-security/trans_sec/device_software/send_packets.py -y 0 -i 0.005 -it 1 -itd 0 -z gateway1-eth1 -sa 0000:0000:0000:0000:0000:0001:0001:0002 -r 0000:0000:0000:0000:0000:0002:0001:0002 -sp 9648 -p 3074 -c 1 -e 00:00:00:00:01:01 -s 00:00:00:01:01:00 -pr TCP -m 'TCP INT IPv6' - -# Without INT -sudo python /home/ubuntu/transparent-security/trans_sec/device_software/send_packets.py -y 0 -i 0.005 -it 1 -itd 0 -z gateway1-eth1 -sa 192.168.1.2 -r 10.0.1.8 -sp 9648 -p 3074 -c 1 -e 00:00:00:00:05:05 -s 00:00:00:01:01:00 -pr UDP -m 'UDP Normal IPv4' -sudo python /home/ubuntu/transparent-security/trans_sec/device_software/send_packets.py -y 0 -i 0.005 -it 1 -itd 0 -z gateway1-eth1 -sa 192.168.1.2 -r 10.0.1.8 -sp 9648 -p 3074 -c 1 -e 00:00:00:00:05:05 -s 00:00:00:01:01:00 -pr TCP -m 'TCP Normal IPv4' - -sudo python /home/ubuntu/transparent-security/trans_sec/device_software/send_packets.py -y 0 -i 0.005 -it 1 -itd 0 -z gateway1-eth1 -sa 0000:0000:0000:0000:0000:0001:0001:0002 -r 0000:0000:0000:0000:0000:0002:0001:0002 -sp 9648 -p 3074 -c 1 -e 00:00:00:00:05:05 -s 00:00:00:01:01:00 -pr UDP -m 'UDP Normal IPv6' -sudo python /home/ubuntu/transparent-security/trans_sec/device_software/send_packets.py -y 0 -i 0.005 -it 1 -itd 0 -z gateway1-eth1 -sa 0000:0000:0000:0000:0000:0001:0001:0002 -r 0000:0000:0000:0000:0000:0002:0001:0002 -sp 9648 -p 3074 -c 1 -e 00:00:00:00:05:05 -s 00:00:00:01:01:00 -pr TCP -m 'TCP Normal IPv6' - - -# Commands for sniffing the important interfaces while generating the packets above -sudo tcpdump -i gateway1-eth1 -w gateway1-eth1.pcap -sudo tcpdump -i gateway1-eth2 -w gateway1-eth2.pcap -sudo tcpdump -i aggregate-eth1 -w aggregate-eth1.pcap -sudo tcpdump -i aggregate-eth2 -w aggregate-eth2.pcap -sudo tcpdump -i core-eth2 -w core-eth2.pcap -sudo tcpdump -i core-eth3 -w core-eth3.pcap diff --git a/bin/send.py b/bin/send.py deleted file mode 100755 index 1c55972c..00000000 --- a/bin/send.py +++ /dev/null @@ -1,130 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import argparse -import logging -import random -import socket -from logging import getLogger, basicConfig -from time import sleep - -from scapy.all import get_if_list, get_if_hwaddr, get_if_addr -from scapy.layers.inet import IP, UDP, TCP -from scapy.layers.l2 import Ether -from scapy.sendrecv import sendp - -# Logger stuff -logger = getLogger('send') - -FORMAT = '%(levelname)s %(asctime)-15s %(filename)s %(message)s' - - -def get_if(target): - iface = None - logger.error(target) - for i in get_if_list(): - logger.info(i.find(target)) - if i.find(target) >= 0: - iface = i - break - if not iface: - logger.error('Cannot find %s interface' % target) - exit(1) - return iface - - -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument('-d', '--duration', help='Number of seconds to run', - type=int, required=True) - parser.add_argument('-i', '--interval', - help='How often to send packets in seconds', - type=float, required=True) - parser.add_argument('-y', '--delay', help='Delay before starting run', - type=int, required=False, default=0) - parser.add_argument('-r', '--destination', help='Destination IPv4 address', - required=True) - parser.add_argument('-sa', '--source-addr', help='Source address') - parser.add_argument('-sp', '--source-port', type=int, - help='Source port else it will be random') - parser.add_argument('-p', '--port', help='Destination port', type=int, - required=True) - parser.add_argument('-m', '--msg', help='Message to send', required=True) - parser.add_argument('-l', '--loglevel', - help='Log Level defaults ' - 'to INFO', - required=False, default='INFO') - parser.add_argument('-f', '--logfile', - help='File to log to defaults to console', - required=False, default=None) - parser.add_argument('-z', '--interface', - help='Linux named ethernet device. Defaults to eth0', - required=False, default='eth0') - parser.add_argument('-s', '--switch_ethernet', - help='Switch Ethernet Interface. Defaults to ' - 'ff:ff:ff:ff:ff:ff', - required=False, default='ff:ff:ff:ff:ff:ff') - parser.add_argument('-t', '--tcp', dest='tcp', action='store_true', - required=False) - args = parser.parse_args() - return args - - -def device_send(args): - numeric_level = getattr(logging, args.loglevel.upper(), None) - basicConfig(format=FORMAT, level=numeric_level, filename=args.logfile) - addr = socket.gethostbyname(args.destination) - count = int((args.duration - args.delay) / args.interval) - interface = get_if(args.interface) - logger.info('Delaying %d seconds' % args.delay) - sleep(args.delay) - - logger.info( - 'sending %s packets at %s sec/packet on interface %r to %s for %s ' - 'seconds', - count, args.interval, interface, args.destination, - (args.duration - args.delay)) - logger.info('SRC ADDR for interface %s - %s', interface, - get_if_hwaddr(interface)) - pkt = Ether(src=get_if_hwaddr(interface), dst=args.switch_ethernet) - logger.info('packet Ether obj - %s', pkt) - - if args.source_addr: - src_ip = args.source_addr - else: - src_ip = get_if_addr(interface) - - if args.source_port: - src_port = args.source_port - else: - src_port = random.randint(49152, 65535) - - if args.tcp: - pkt = pkt / IP(dst=addr, src=src_ip) / TCP(dport=args.port, - sport=src_port) / args.msg - else: - pkt = pkt / IP(dst=addr, src=src_ip) / UDP(dport=args.port, - sport=src_port) / args.msg - - pkt.show2() - sendp(pkt, iface=interface, verbose=False, count=count, - inter=args.interval) - logger.info('Done') - return - - -if __name__ == '__main__': - logger.info('Starting Send') - cmd_args = get_args() - device_send(cmd_args) diff --git a/bin/write_to_clone.py b/bin/write_to_clone.py deleted file mode 100755 index 175aac2d..00000000 --- a/bin/write_to_clone.py +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import argparse -import logging -import sys - -from trans_sec.p4runtime_lib.p4rt_switch import P4RuntimeSwitch -from trans_sec.p4runtime_lib.helper import P4InfoHelper -from trans_sec.switch import SwitchConnection - -logger = logging.getLogger('insert_p4_clone_session_entry') - - -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument('-a', '--grpc_addr', type=str, required=True, - help='Switch grpc host:port') - parser.add_argument('-d', '--dev_id', type=int, required=False, default=0, - help='The device ID (default 0)') - parser.add_argument( - '-p', '--p4-info-fpath', type=str, required=False, default=0, - help='The file path of the switch associated p4info file') - parser.add_argument('-l', '--log-dir', type=str, required=False, - default=None) - parser.add_argument('-lf', '--log-file', type=str, required=False, - default='insert_p4_table.log') - parser.add_argument('-c', '--clone', type=str, required=False, - default='True', help='Build clone entry') - parser.add_argument('-ce', '--clone-egress', type=int, required=False, - default=0, help='Clone egress port') - return parser.parse_args() - - -if __name__ == '__main__': - """ - Inserts clone entries based on the table config file - """ - args = get_args() - if args.log_dir and args.log_file: - log_file = '{}/{}'.format(args.log_dir, args.log_file) - logging.basicConfig(stream=sys.stdout, level=logging.DEBUG, - filename=log_file) - else: - logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) - - logger.info('Connecting to BMV2 Switch #[%s] at [%s] loaded with P4 [%s]', - args.dev_id, args.grpc_addr, args.p4_info_fpath) - p4info_helper = P4InfoHelper(args.p4_info_fpath) - - # TODO/FIXME - This will break and need to know which type of switch - # connection to start - switch = SwitchConnection( - name='test', address=args.grpc_addr, device_id=args.dev_id) - switch.master_arbitration_update() - - if args.clone == 'True': - clone_entry = p4info_helper.build_clone_entry(args.clone_egress) - logger.info('Inserting clone entry [%s]', clone_entry) - switch.write_clone_entries(clone_entry) - else: - clone_entry = p4info_helper.build_clone_entry(args.clone_egress) - logger.info('Deleting clone entry [%s]', clone_entry) - switch.delete_clone_entries(clone_entry) diff --git a/bin/write_to_p4_table.py b/bin/write_to_p4_table.py deleted file mode 100755 index 4df302ff..00000000 --- a/bin/write_to_p4_table.py +++ /dev/null @@ -1,146 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import argparse -import logging -import sys - -import yaml - -from trans_sec.p4runtime_lib.gateway_switch import GatewaySwitch -from trans_sec.p4runtime_lib.aggregate_switch import AggregateSwitch -from trans_sec.p4runtime_lib.core_switch import CoreSwitch -from trans_sec.p4runtime_lib.helper import P4InfoHelper - -logger = logging.getLogger('insert_p4_table_entry') - - -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument('-s', '--switch-name', required=True, - help='The switch name to control') - parser.add_argument('-t', '--topo', required=True, - help='The location to the topology file') - parser.add_argument('-n', '--ingress', required=True, - help='The P4 ingress classname') - parser.add_argument( - '-i', '--insert', type=str, required=False, default='True', - help='When true insert record else delete (default "True")') - parser.add_argument('-tc', '--table-config', - help='Table insertion config file', - type=str, required=False) - parser.add_argument('-e', '--election-id', - help='Table insertion config file', - type=int, required=False, default=0) - parser.add_argument('-l', '--log-dir', type=str, required=False, - default=None) - parser.add_argument('-lf', '--log-file', type=str, required=False, - default='insert_p4_table.log') - return parser.parse_args() - - -def read_yaml_file(config_file_path): - """ - Reads a yaml file and returns a dict representation of it - :return: a dict of the yaml file - """ - logger.debug('Attempting to load configuration file - ' + config_file_path) - config_file = None - try: - with open(config_file_path, 'r') as config_file: - config = yaml.safe_load(config_file) - logger.info('Loaded configuration') - return config - finally: - if config_file: - logger.info('Closing configuration file') - config_file.close() - - -if __name__ == '__main__': - """ - Inserts entries into P4 tables based on the entries within table config - file - """ - args = get_args() - if args.log_dir and args.log_file: - log_file = '{}/{}'.format(args.log_dir, args.log_file) - logging.basicConfig(level=logging.DEBUG, filename=log_file) - else: - logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) - - table_entry_config = read_yaml_file(args.table_config) - topo = read_yaml_file(args.topo) - sw_info = topo['switches'][args.switch_name] - - logger.info('Connecting to BMV2 Switch #[%s]', sw_info) - p4info_helper = P4InfoHelper(sw_info['runtime_p4info']) - - for entry_config in table_entry_config: - logger.info('Entry config - [%s]', entry_config) - logger.info('Writing table entry - [%s] for insert - [%s]', - entry_config, args.insert) - - switch = None - if sw_info['type'] == 'gateway': - switch = GatewaySwitch(sw_info) - elif sw_info['type'] == 'aggregate': - switch = AggregateSwitch(sw_info) - elif sw_info['type'] == 'core': - switch = CoreSwitch(sw_info) - - if not switch: - raise Exception('Switch type of [%s] is not supported', - sw_info['type']) - switch.master_arbitration_update() - - match_fields = dict() - if 'match_fields' in entry_config: - logger.info('Match fields obj - [%s]', - entry_config['match_fields']) - for key, value in entry_config['match_fields'].items(): - if isinstance(value, list) or isinstance(value, tuple): - logger.debug('Match fields list - [%s]', value) - match_fields[key] = (value[0], value[1]) - else: - logger.debug('Match fields list - [%s]', value) - match_fields[key] = value - - if args.insert == 'True': - entries = switch.get_match_values(entry_config['table_name']) - logger.info('Existing table entries - {}'.format(entries)) - logger.info( - 'Entry configuration for table entry - [%s] and match fields ' - '- [%s]', entry_config, match_fields) - table_entry = p4info_helper.build_table_entry( - table_name=entry_config['table_name'], - match_fields=match_fields, - action_name=entry_config.get('action_name'), - action_params=entry_config.get('action_params'), - ) - logger.debug( - 'Writing table entry to table [%s], with action name - [%s], ' - 'match fields - [%s], action_params - [%s]', - entry_config['table_name'], entry_config.get('action_name'), - match_fields, entry_config.get('action_params')) - switch.write_table_entry(table_entry, - election_high=args.election_id, - election_low=args.election_id + 1) - else: - table_entry = p4info_helper.build_table_entry( - table_name=entry_config['table_name'], - match_fields=entry_config.get('match_fields'), - ) - logger.info('Deleting table entry') - switch.delete_table_entry(table_entry) diff --git a/playbooks/scenarios/test_cases/write_clone_entry.yml b/playbooks/scenarios/test_cases/write_clone_entry.yml deleted file mode 100644 index 7df48b88..00000000 --- a/playbooks/scenarios/test_cases/write_clone_entry.yml +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Simple scenario where packets are sent through 3 devices and only the last -# one will be demonstrating dropped packets ---- -- hosts: localhost - gather_facts: no - vars: - topo_dict: "{{ lookup('file','{{ topo_file_loc }}') | from_yaml }}" - switch: "{{ topo_dict.switches.core }}" - log_dir: "{{ log_dir }}" - log_file: write_to_clone.log - clone: "{{ clone_entry | default('True') }}" - tasks: - - name: Create clone session entry command - set_fact: - add_clone_entry_cmd: > - {{ trans_sec_dir }}/bin/write_to_clone.py - -a {{ switch.grpc }} -d {{ switch.id }} -p {{ switch.runtime_p4info }} -l {{ log_dir }} - -lf {{ log_file }} -c {{ clone }} -ce {{ switch.clone_egress }} - - - name: Create log directory {{ log_dir }} - file: - path: "{{ log_dir }}" - state: directory - - - name: Build clone entry with command [{{ add_clone_entry_cmd }}] - command: "{{ add_clone_entry_cmd }}" - register: cmd_out - changed_when: cmd_out is not failed diff --git a/playbooks/scenarios/test_cases/write_to_p4_table.yml b/playbooks/scenarios/test_cases/write_to_p4_table.yml deleted file mode 100644 index 00154d76..00000000 --- a/playbooks/scenarios/test_cases/write_to_p4_table.yml +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# integration test case where one host creates a packet sniffer and another -# sends packets. The validation routine counts the number received via the -# receiver's log file ---- - -# Adds a table entry to a P4 switch -- hosts: localhost - gather_facts: no - vars: - table_entry_config: "{{ te_conf_file }}" - log_dir: "{{ p4_te_log_dir }}" - log_file: "{{ p4_te_log_file }}" - insert_flag: "{{ p4_te_insert | default('True') }}" - template_file: ../templates/table_entry.yml.j2 - table_action: "{{ p4_table_action }}" - ingress_table: "{{ p4_ingress_table_flag | default('True') }}" - execute: "{{ execute_insert | default(True) }}" - tasks: - - name: Execute when - block: - - name: Create config {{ table_entry_config }} - template: - src: "{{ template_file }}" - dest: "{{ table_entry_config }}" - backup: yes - - - name: Creating add table command - set_fact: - add_table_entry_cmd: > - {{ trans_sec_dir }}/bin/write_to_p4_table.py - -s {{ switch.name }} - -t {{ topo_file_loc }} - -tc {{ table_entry_config }} - -l {{ log_dir }} - -lf {{ log_file }} - -n {{ ingress_table }} - - - name: Create log directory {{ log_dir }} - file: - path: "{{ log_dir }}" - state: directory - - - name: Applying requested table entry with command [{{ add_table_entry_cmd }}] with insert_flag - [{{ insert_flag }}] - command: "{{ add_table_entry_cmd }} -i {{ insert_flag }}" - register: cmd_out - changed_when: cmd_out is not failed - ignore_errors: yes - - - name: Deleting requested table entry with command [{{ add_table_entry_cmd }}] and election-id 2 - command: "{{ add_table_entry_cmd }} -i False" - when: cmd_out is failed and insert_flag == "True" - - - name: Try table entry with command [{{ add_table_entry_cmd }}] with election-id 2 - command: "{{ add_table_entry_cmd }} -e 2 -i {{ insert_flag }}" - when: cmd_out is failed - when: execute|bool diff --git a/trans_sec/bfruntime_lib/gateway_switch.py b/trans_sec/bfruntime_lib/gateway_switch.py deleted file mode 100644 index 820fcb69..00000000 --- a/trans_sec/bfruntime_lib/gateway_switch.py +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright (c) 2020 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Originally copied from: -# -# Copyright 2017-present Open Networking Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import logging -from abc import ABC - -from trans_sec.bfruntime_lib.bfrt_switch import BFRuntimeSwitch - -logger = logging.getLogger('gateway_switch') - - -class GatewaySwitch(BFRuntimeSwitch, ABC): - def __init__(self, sw_info, client_id=0, is_master=True): - """ - Construct Switch class to control BMV2 switches running gateway.p4 - """ - super(self.__class__, self).__init__(sw_info, client_id, is_master) - self.nat_udp_ports = set() - self.nat_tcp_ports = set() - self.tcp_port_count = 1 - self.udp_port_count = 1 - - def add_data_inspection(self, dev_id, dev_mac): - raise NotImplementedError - - def write_multicast_entry(self, hosts): - raise NotImplementedError From e7dba1de175964c3414f2ae7a7489bc8f6d80001 Mon Sep 17 00:00:00 2001 From: spisarski Date: Fri, 6 Aug 2021 10:57:47 -0600 Subject: [PATCH 16/40] pylint fix --- trans_sec/controller/aggregate_controller.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/trans_sec/controller/aggregate_controller.py b/trans_sec/controller/aggregate_controller.py index d4c0cba7..c3549be2 100755 --- a/trans_sec/controller/aggregate_controller.py +++ b/trans_sec/controller/aggregate_controller.py @@ -11,15 +11,14 @@ # See the License for the specific language governing permissions and # limitations under the License. from logging import getLogger +from trans_sec.bfruntime_lib.aggregate_switch import ( + AggregateSwitch as BFRTSwitch) from trans_sec.controller.abstract_controller import AbstractController from trans_sec.controller.ddos_sdn_controller import AGG_CTRL_KEY from trans_sec.utils import tps_utils logger = getLogger('aggregate_controller') -from trans_sec.bfruntime_lib.aggregate_switch import ( - AggregateSwitch as BFRTSwitch) - class AggregateController(AbstractController): """ From 866962a3031bf5bfb234067f6dd376fab47a7506 Mon Sep 17 00:00:00 2001 From: spisarski Date: Sat, 7 Aug 2021 00:16:25 -0600 Subject: [PATCH 17/40] Fixed tofino build by updating all apt packages and cache after env_build components had been installed. --- automation/env-build/aws-inst.tf | 12 +++++------- automation/env-build/variables.tf | 5 +---- automation/p4/tofino/variables.tf | 4 ++-- playbooks/general/final_env_setup.yml | 19 ++++++++++++++++++- 4 files changed, 26 insertions(+), 14 deletions(-) diff --git a/automation/env-build/aws-inst.tf b/automation/env-build/aws-inst.tf index 73f6d9d8..77a623e1 100644 --- a/automation/env-build/aws-inst.tf +++ b/automation/env-build/aws-inst.tf @@ -22,17 +22,15 @@ locals { ] py2 = [ "sudo apt-get update", - "sudo apt-get install python2.7 aptitude -y", - ] - ae = [ - "sudo rm -f /usr/bin/python", - "sudo ln -s /usr/bin/python2.7 /usr/bin/python" + "sudo apt-get install python2.7 -y", + "sudo apt-get update", + "sudo apt-get install aptitude -y", ] - inline_scripts = var.env_type == "ae" ? local.ae : var.env_type == "tofino" ? local.py2 : var.env_type == "siddhi" ? local.none : local.py3 + inline_scripts = var.env_type == "tofino" ? local.py2 : local.none } resource "aws_instance" "transparent-security-build-img" { - ami = var.env_type == "ae" ? var.centos7_ami : var.ubuntu_version == "20" ? var.base_20_ami : var.ubuntu_version == "18" ? var.base_18_ami : var.base_16_ami + ami = var.ubuntu_version == "20" ? var.base_20_ami : var.ubuntu_version == "18" ? var.base_18_ami : var.base_16_ami instance_type = var.instance_type key_name = aws_key_pair.transparent-security-mini-pk.key_name diff --git a/automation/env-build/variables.tf b/automation/env-build/variables.tf index a04a1d32..573cba2c 100644 --- a/automation/env-build/variables.tf +++ b/automation/env-build/variables.tf @@ -24,7 +24,7 @@ variable "bf_sde_s3_bucket" {default = "null"} # Optional Variables variable "bf_sde_version" {default = "9.2.0"} variable "bf_sde_profile" {default = "p416_examples_profile"} -variable "create_ami" {default = "yes"} +variable "create_ami" {default = "no"} variable "public_key_file" {default = "~/.ssh/id_rsa.pub"} variable "private_key_file" {default = "~/.ssh/id_rsa"} variable "sudo_user" {default = "ubuntu"} @@ -32,9 +32,6 @@ variable "ubuntu_version" {default = "18"} variable "siddhi_map_p4_trpt_version" {default = "master"} variable "python_version" {default = "3.6"} -# snaps-hcp image -variable "centos7_ami" {default="ami-01ed306a12b7d1c96"} - # ubuntu 16 variable "base_16_ami" {default = "ami-08692d171e3cf02d6"} # ubuntu 18 diff --git a/automation/p4/tofino/variables.tf b/automation/p4/tofino/variables.tf index 6d1ace96..0bbb3945 100644 --- a/automation/p4/tofino/variables.tf +++ b/automation/p4/tofino/variables.tf @@ -32,11 +32,11 @@ variable "tofino" { default = { sde_version = "9.2.0" p4rt_ami = "ami-043e6714f3d0863f2" - bfrt_ami = "ami-01a5ff54de23b6739" + bfrt_ami = "ami-090b409398f016b84" } } -variable "siddhi_ae_ami" {default = "ami-05bb35a9f06141f6b"} +variable "siddhi_ae_ami" {default = "ami-07e5e73831d02dbe9"} variable "switch_instance_type" {default = "t2.2xlarge"} variable "ae_instance_type" {default = "t2.2xlarge"} diff --git a/playbooks/general/final_env_setup.yml b/playbooks/general/final_env_setup.yml index 1e383a52..bc7159a5 100644 --- a/playbooks/general/final_env_setup.yml +++ b/playbooks/general/final_env_setup.yml @@ -19,6 +19,7 @@ - name: Install python3-pip and wireshark become: yes apt: + update_cache: yes name: - python3-pip - wireshark-qt @@ -34,8 +35,24 @@ src: ../../requirements.txt dest: ~/tps-py-requirements.txt - - name: Install TPS Python requirements + - name: Install TPS pip requirements command: sudo pip install -r ~/tps-py-requirements.txt + register: pip_install + ignore_errors: yes + + - name: Install TPS pip3 requirements + command: sudo pip3 install -r ~/tps-py-requirements.txt + when: pip_install is failed + + - name: Final upgrade of all apt packages + become: yes + apt: + upgrade: yes + + - name: Final apt cache update + become: yes + apt: + update_cache: yes - name: Disable unattended-upgrades become: yes From 45113b88ba8851840fac093b7c17bced7f7fc7ac Mon Sep 17 00:00:00 2001 From: spisarski Date: Mon, 9 Aug 2021 12:33:16 -0600 Subject: [PATCH 18/40] Initial doc improvements --- automation/env-build/aws-inst.tf | 2 +- automation/env-build/variables.tf | 2 +- automation/p4/tofino/aws-insts.tf | 2 +- automation/p4/tofino/variables.tf | 5 +- docs/ae/SIDDHI_AE_SETUP.md | 28 +-- docs/env_build/CREATE_AUTOMATION_IMAGES.md | 112 ++++++++++++ docs/terraform/automation-base.example.tfvars | 4 + .../siddhi-maven-build.example.tfvars | 4 + docs/terraform/tofino-build.example.tfvars | 4 + docs/terraform/tofino-int.example.tfvars | 5 + docs/tofino/RUN_CI_AUTOMATION.md | 89 +++++++++ docs/tofino/automation.md | 170 ------------------ playbooks/env-build/siddhi/env_build.yml | 2 +- .../{setup.yml => setup_siddhi_maven.yml} | 0 14 files changed, 240 insertions(+), 189 deletions(-) create mode 100644 docs/env_build/CREATE_AUTOMATION_IMAGES.md create mode 100644 docs/terraform/automation-base.example.tfvars create mode 100644 docs/terraform/siddhi-maven-build.example.tfvars create mode 100644 docs/terraform/tofino-build.example.tfvars create mode 100644 docs/terraform/tofino-int.example.tfvars create mode 100644 docs/tofino/RUN_CI_AUTOMATION.md delete mode 100644 docs/tofino/automation.md rename playbooks/siddhi/{setup.yml => setup_siddhi_maven.yml} (100%) diff --git a/automation/env-build/aws-inst.tf b/automation/env-build/aws-inst.tf index 77a623e1..32dc1311 100644 --- a/automation/env-build/aws-inst.tf +++ b/automation/env-build/aws-inst.tf @@ -75,7 +75,7 @@ aws_access_key=${var.access_key} aws_secret_key=${var.secret_key} bf_sde_version=${var.bf_sde_version} bf_sde_profile=${var.bf_sde_profile} -bf_sde_s3_bucket=${var.bf_sde_s3_bucket} +bf_sde_s3_bucket=${var.bf_sde_s3_bucket == null ? "n/a" : var.bf_sde_s3_bucket} remote_scripts_dir=${var.remote_scripts_dir} python_version=${var.python_version} ubuntu_version=${var.ubuntu_version} diff --git a/automation/env-build/variables.tf b/automation/env-build/variables.tf index 573cba2c..bcd9b88f 100644 --- a/automation/env-build/variables.tf +++ b/automation/env-build/variables.tf @@ -19,7 +19,7 @@ variable "ec2_region" {} variable "env_type" {default = "tofino"} # Dependency version only for tofino environments -variable "bf_sde_s3_bucket" {default = "null"} +variable "bf_sde_s3_bucket" {default = null} # Optional Variables variable "bf_sde_version" {default = "9.2.0"} diff --git a/automation/p4/tofino/aws-insts.tf b/automation/p4/tofino/aws-insts.tf index 448d7ea3..bf53efb6 100644 --- a/automation/p4/tofino/aws-insts.tf +++ b/automation/p4/tofino/aws-insts.tf @@ -12,7 +12,7 @@ # limitations under the License. locals { - ami = var.p4_arch == "tna" ? var.tofino.bfrt_ami : var.tofino.p4rt_ami + ami = var.tofino.bfrt_ami dflt_start_script = ["sudo echo 'tps-orchestrator/controller' > ~/motd"] // Placeholder for custom startup options for BFRT images tofino_img_start_script = local.dflt_start_script diff --git a/automation/p4/tofino/variables.tf b/automation/p4/tofino/variables.tf index 0bbb3945..56ead961 100644 --- a/automation/p4/tofino/variables.tf +++ b/automation/p4/tofino/variables.tf @@ -30,9 +30,8 @@ variable "availability_zone" {default = "us-west-2a"} variable "tofino" { default = { - sde_version = "9.2.0" - p4rt_ami = "ami-043e6714f3d0863f2" - bfrt_ami = "ami-090b409398f016b84" + sde_version = {default = "9.2.0"} + bfrt_ami = {default = "ami-090b409398f016b84"} } } diff --git a/docs/ae/SIDDHI_AE_SETUP.md b/docs/ae/SIDDHI_AE_SETUP.md index 26ba879f..64262098 100644 --- a/docs/ae/SIDDHI_AE_SETUP.md +++ b/docs/ae/SIDDHI_AE_SETUP.md @@ -21,8 +21,9 @@ Kubernetes where the siddhi-operator has been installed. ## Setup and run -There are several things that must be done to the host prior to attempting to -start the Siddhi TPS AE running as a standalone Java application +There are several methods that can be used to run Siddhi scripts. + +### Java application - Install Java 8+ (CI is running OpenJDK 11.0.11) - Install Git client (apt install git) @@ -35,16 +36,19 @@ start the Siddhi TPS AE running as a standalone Java application - Execute ```bash +cd {siddhi-map-p4-trpt directory} mvn exec:java -Dexec.mainClass=io.siddhi.extension.map.p4.StartSiddhiRuntime \ --Dexec.args="/etc/transparent-security/convert_trpt.siddhi \ -/etc/transparent-security/simple_ddos_detection.siddhi \ -/etc/transparent-security/simple_ddos_clear.siddhi \ -" -f pom.xml +-Dexec.args="{siddhi-script-location} {siddhi-script-location} ..." -f pom.xml ``` -## Making changes - -The StartSiddhiRuntime Java main() simply starts a Siddhi Runtime then takes -the file contents of each of the arguments and loads them into the engine. -Therefore, changes can be made to any or all of the siddhi files and/or other -Siddhi scripts can be added/deleted. +### Kubernetes +Other than a Siddhi runtime server or Java application, another means to deploy +Siddhi scripts is on Kubernetes with the +[siddhi-operator](https://github.com/siddhi-io/siddhi-operator). + +- Install siddhi-operator +- Apply CRDs + - kafka-operator.yml + - parse-trpt-udp.yml + - eval-pkt-trpt.yml + - eval-drop-trpt.yml diff --git a/docs/env_build/CREATE_AUTOMATION_IMAGES.md b/docs/env_build/CREATE_AUTOMATION_IMAGES.md new file mode 100644 index 00000000..eb417150 --- /dev/null +++ b/docs/env_build/CREATE_AUTOMATION_IMAGES.md @@ -0,0 +1,112 @@ +# transparent-security automation for the tofino-model + +The scripts outlined here have been designed to be executed within a CI server + + +### Getting started +Below are the variables used by the env_build CI automation scripts. + +| Variable | Description | Type | Example | +|------------------|-------------------------------------------------------------------------------------------------------------------------------------------|--------|---------------------------------------------------------| +| build_id | This value must be unique to ensure multiple jobs can be run simultaneously from multiple hosts | string | build_id = "example-1" | +| access_key | Amazon EC2 access key | string | access_key = "AKIAIOSFODNN7EXAMPLE" | +| secret_key | Amazon EC2 secret key | string | secret_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" | +| ec2_region | Amazon EC2 region | string | ec2_region = "us-west-2" (default) | +| public_key_file | Used to inject into the VM for SSH access with the user'ubuntu' (defaults to ~/.ssh/id_rsa.pub) | string | public_key_file = "~/.ssh/id_rsa.pub" (default) | +| private_key_file | Used to access the VM via SSH with the user 'ubuntu' (defaults to ~/.ssh/id_rsa) | string | private_key_file = "~/.ssh/id_rsa" (default) | +| env_type | The type of environment being built (only used for creating the environment) | string | env_type = {"tofino"|"siddhi"} (default="tofino") | +| create_ami | When 'yes', the Terraform script will create a new AMI | string | create_ami = "no" (default) | +| bf_sde_version | When this value will be used to obtaining the BF-SDE version | string | bf_sde_version = "9.2.0" (default) | +| bf_sde_profile | The BF-SDE profile name to install | string | bf_sde_profile = "p416_examples_profile" (default) | +| bf_sde_s3_bucket | The S3 bucket in which the BF SDE tar file lives (named bf-sde-{version}.tar) | string | bf_sde_s3_bucket = "EXAMPLE_S3_BUCKET" | + +### Build AMIs for running the tofino-model and Siddhi AE on AWS + +#### Create Tofino BF-SDE EC2 image with Terraform + +##### Install Terraform +https://www.terraform.io/downloads.html + +##### Upload BF-SDE archive +Create s3 bucket and upload the BF-SDE tar/tgz using the same AWS credentials. + +##### Configure +Please see the following example Terraform Variable files to create the files +to hold the Terraform program arguments via the --var-file command line option +in the "Execute" section below. + +- [my-automation-base.tfvars template](../terraform/automation-base.example.tfvars) +- [my-tofino-build.tfvars template](../terraform/tofino-build.example.tfvars) +- [my-siddhi-maven-build.tfvars template](../terraform/siddhi-maven-build.example.tfvars) + +##### Build Tofino image +This step will create an VM on AWS, install all Tofino 9.2.0 dependencies, +then create an AMI. (note: this process will take ~90 min to complete) + +```bash +cd transparent-security/automation/p4/env-build +terraform init +terraform apply -auto-approve -var-file="my-automation-base.tfvars -var-file="my-tofino-build.tfvars" -var build_id={i.e. "my-build-x"}" +``` + +##### Build Siddhi Maven image +This step will create an VM on AWS, install everything required to run Siddhi +with the udp & p4-trpt extensions and Kafka. (note: this image is only required +for the "lab_trial" p4/tofino scenario and will take ~30 min to complete) + +```bash +cd transparent-security/automation/p4/env-build +terraform init +terraform apply -auto-approve -var-file="my-automation-base.tfvars -var-file="my-siddhi-maven-build.tfvars" -var build_id={i.e. "my-build-y"}" +``` + +##### Sample output from the environment build terraform script +Sample Output: + +```bash +. +. +. +Apply complete! Resources: 5 added, 0 changed, 0 destroyed. + +Outputs: + +ami-id = ami-xxx +ip = xx.xx.xx.xx +``` + +Save the ami-ids from each run to be used for running the p4/tofino automation scripts. + +### Remove the AMI for terraform state + +Remove the AMI from terraform state so that it will remain after destroying the +VM should you want to continue to use this image. + +```bash +terraform state rm aws_ami_from_instance.transparent-security-env-build +Removed aws_ami_from_instance.transparent-security-env-build +Successfully removed 1 resource instance(s). +``` + +### Clean up the VM used to create the AMI + +This step will remove everything except the AMI that was used to create the VM +when the call above has been made. + +```bash +terraform destroy -auto-approve -var-file="/path/to/my-tofino.tfvars -var-file="(my-tofino-build.tfvars|my-siddhi-maven-build.tfvars)" +``` + +Sample output: + +```bash +. +. +. +aws_key_pair.transparent-security-mini-pk: Destruction complete after 0s +aws_security_group.transparent-security-img-sg: Destruction complete after 0s + +Destroy complete! Resources: 5 destroyed. + +Process finished with exit code 0 +``` diff --git a/docs/terraform/automation-base.example.tfvars b/docs/terraform/automation-base.example.tfvars new file mode 100644 index 00000000..d799aad0 --- /dev/null +++ b/docs/terraform/automation-base.example.tfvars @@ -0,0 +1,4 @@ +# EC2 credentials +access_key = "myAccessKey" +secret_key = "mySecretKey" +ec2_region = "my-region" diff --git a/docs/terraform/siddhi-maven-build.example.tfvars b/docs/terraform/siddhi-maven-build.example.tfvars new file mode 100644 index 00000000..5a515232 --- /dev/null +++ b/docs/terraform/siddhi-maven-build.example.tfvars @@ -0,0 +1,4 @@ +# EC2 credentials for Transparent Security CI +env_type = "siddhi" +ubuntu_version = "20" +create_ami = "yes" diff --git a/docs/terraform/tofino-build.example.tfvars b/docs/terraform/tofino-build.example.tfvars new file mode 100644 index 00000000..73cca79e --- /dev/null +++ b/docs/terraform/tofino-build.example.tfvars @@ -0,0 +1,4 @@ +# EC2 credentials for Transparent Security CI +bf_sde_s3_bucket = "my-s3-bucket" +env_type = "tofino" +create_ami = "yes" diff --git a/docs/terraform/tofino-int.example.tfvars b/docs/terraform/tofino-int.example.tfvars new file mode 100644 index 00000000..e80362bd --- /dev/null +++ b/docs/terraform/tofino-int.example.tfvars @@ -0,0 +1,5 @@ +tofino = { + sde_version = "9.2.0" + bfrt_ami = "ami-xxx" +} +siddhi_ae_ami = "ami-yyy" diff --git a/docs/tofino/RUN_CI_AUTOMATION.md b/docs/tofino/RUN_CI_AUTOMATION.md new file mode 100644 index 00000000..dbe4f4aa --- /dev/null +++ b/docs/tofino/RUN_CI_AUTOMATION.md @@ -0,0 +1,89 @@ +# transparent-security automation for the tofino-model + +The scripts outlined here can be executed within a CI server or locally during +development + +### Getting started +Below are the variables used by the tofino CI automation scripts. + +| Variable | Description | Type | Example | +|------------------|-------------------------------------------------------------------------------------------------------------------------------------------|--------|---------------------------------------------------------| +| build_id | This value must be unique to ensure multiple jobs can be run simultaneously from multiple hosts | string | build_id = "example-1" | +| access_key | Amazon EC2 access key | string | access_key = "AKIAIOSFODNN7EXAMPLE" | +| secret_key | Amazon EC2 secret key | string | secret_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" | +| ec2_region | Amazon EC2 region | string | ec2_region = "us-west-2" (default) | +| public_key_file | Used to inject into the VM for SSH access with the user'ubuntu' (defaults to ~/.ssh/id_rsa.pub) | string | public_key_file = "~/.ssh/id_rsa.pub" (default) | +| private_key_file | Used to access the VM via SSH with the user 'ubuntu' (defaults to ~/.ssh/id_rsa) | string | private_key_file = "~/.ssh/id_rsa" (default) | +| bf_sde_version | When this value will be used to obtaining the BF-SDE version | string | bf_sde_version = "9.2.0" (default) | + +### Run terraform to launch the cluster on AWS. + +##### Configure +Please see the following example Terraform Variable files to create the files +to hold the Terraform program arguments via the --var-file command line option +in the "Execute" section below. + +- [my-automation-base.tfvars template](../terraform/automation-base.example.tfvars) +- [my-tofino-int.tfvars template](../terraform/tofino-int.example.tfvars) + + +```bash +cd transparent-security/automation/p4/tofino +terraform init +terraform apply -auto-approve -var-file="my-automation-base.tfvars -var-file="my-tofino-build.tfvars" -var scenario_name=(aggregate|core|lab_trial)" +``` + +### SSH into orchestrator/controller machine + +Use the SSH keys indicated in the variable file to login to the VM. +```bash +# from transparent-security/automation/p4/tofino directory +ssh -i { key file } ubuntu@$(terraform output ip) +``` + +### What is deployed +Controller/Orchestrator node with outside access + +#### scenario_name=lab_trial +* 1 orchestrator/controller node (Performs deployment and runs SDN controller) +* 4 network nodes (Standard Linux VMs) +* 2 Tofino switches (Linux VMs with the BF-SDE running the P4 programs) + +###### Switches (with bf-sde-{version} and transparent-security installed into python runtime) +- core (running core.p4) +- aggregate (running aggregate.p4) + +###### Nodes (vanilla linux with transparent-security installed into python runtime) +- inet (to core) +- ae (to core) +- host1 (to aggregate) +- host2 (to core) + +#### scenario_name=aggregate|core +* 1 orchestrator/controller node +* 2 network nodes +* 1 Tofino switch + +###### Switches (with bf-sde-{version} and transparent-security installed into python runtime) +- runs (aggregate|core).p4 + +###### Nodes (vanilla linux with transparent-security installed into python runtime) +- host1 (southbound server node) +- host2 (northbound server node) + +#### Accessing the switches and nodes from orchestrator +From the orchestrator node, you can gain access to all other nodes and switch VMs +by name with user 'ubuntu': + +```bash +ssh {switch_name | node_name} +``` + +### Cleanup the simulation environment + +This will remove the VM and other artifacts created when it was deployed. + +```bash +# from transparent-security/automation/p4/tofino directory +terraform destroy -auto-approve -var-file="my-automation-base.tfvars -var-file="my-tofino-build.tfvars" -var scenario_name="destroy" +``` diff --git a/docs/tofino/automation.md b/docs/tofino/automation.md deleted file mode 100644 index 8cfe3090..00000000 --- a/docs/tofino/automation.md +++ /dev/null @@ -1,170 +0,0 @@ -# transparent-security automation for the tofino-model - -The scripts outlined here have been designed to be executed within a CI server - - -### Getting started -Please see "Client system setup" in ../BUILD.md - -Please uses these variables instead -Copy the example variable file docs/tofino-example.tfvars to a working -directory and make changes to adapt the file to your local environment. - -| Variable | Description | Type | Example | -|------------------|-------------------------------------------------------------------------------------------------------------------------------------------|--------|---------------------------------------------------------| -| build_id | This value must be unique to ensure multiple jobs can be run simultaneously from multiple hosts | string | build_id = "test-1" | -| access_key | Amazon EC2 access key | string | access_key = "AKIAIOSFODNN7EXAMPLE" | -| secret_key | Amazon EC2 secret key | string | secret_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" | -| ec2_region | Amazon EC2 region | string | ec2_region = "us-west-2" | -| public_key_file | Used to inject into the VM for SSH access with the user'ubuntu' (defaults to ~/.ssh/id_rsa.pub) | string | public_key_file = "~/.ssh/id_rsa.pub" | -| private_key_file | Used to access the VM via SSH with the user 'ubuntu' (defaults to ~/.ssh/id_rsa) | string | private_key_file = "~/.ssh/id_rsa" | -| env_type | The type of environment being built (only used for creating the environment) | string | env_type = "tofino" | - -### Build AMI for running the tofino-model on AWS - -#### Create VM with Terraform - -Create s3 bucket using the same AWS credentials for the BF-SDE tar/tgz -bf_sde_s3_bucket -This step will creat an VM on AWS, install all Tofino 9.2.0 dependencies and create an AMI. - -```bash -cd transparent-security/automation/p4/env-build -terraform init -terraform apply -auto-approve -var-file="/path/to/my-tofino.tfvars -var env_type=tofino" -``` - -Sample Output: - -```bash -aws_key_pair.transparent-security-mini-pk: Creating... -aws_security_group.transparent-security-img-sg: Creating... -aws_key_pair.transparent-security-mini-pk: Creation complete after 1s -. -. -. -Apply complete! Resources: 12 added, 0 changed, 0 destroyed. - -Outputs: - -ami-id = ami-0393652ac3fbc331e -ip = 34.211.114.181 -``` - -Save the ami-id and it to your variables file. - -### Remove the AMI for terraform state - -Remove the AMI from the terraform state so that it will remain after destroying the VM. - -```bash -terraform state rm aws_ami_from_instance.transparent-security-env-build -Removed aws_ami_from_instance.transparent-security-env-build -Successfully removed 1 resource instance(s). -``` - -### Clean up the VM used to create the AMI - -This step will remove everything except the AMI that was used to create the VM. - -```bash -terraform destroy -auto-approve -var-file="/path/to/my-tofino.tfvars" -``` - -Sample output: - -```bash -aws_key_pair.transparent-security-mini-pk: Refreshing state... [id=terraform-20191213203053435500000001] -aws_security_group.transparent-security-img-sg: Refreshing state... [id=sg-057e54e0162c6251a] -. -. -. -Destroy complete! Resources: 4 destroyed. -``` - -## Create a virtual Tofino switch environment (using Terraform) - -Use the environment file created in section 2.4 - -### Run terraform to launch the cluster on AWS. - -Variable file must also contain the following lines as the default is tied to -a CableLabs AWS account: -```hcl-terraform -variable "tofino" { - default = { - sde_version = "9.2.0" - ami = "{{ your ami-version }}" - } -} -``` - -```bash -cd transparent-security/automation/p4/tofino -terraform init -terraform apply -auto-approve -var-file="/path/to/my-tofino.tfvars -var build_id={your unique ID} -var scenario_name=(full|gateway|aggregate|core)" -``` - -### SSH into orchestrator/controller machine - -Use the SSH keys indicated in the variable file to login to the VM. -```bash -# from transparent-security/automation/p4/tofino directory -ssh -i { key file } ubuntu@$(terraform output ip) -``` - -### What is deployed -Controller/Orchestrator node with outside access - -#### scenario_name=full -* 1 orchestrator/controller node (Performs deployment and runs SDN controller) -* 9 network nodes (Standard Linux VMs) -* 5 Tofino switches (Linux VMs with Tofino Model installed running the P4 programs) - -###### Switches (with bf-sde-{version} and transparent-security installed into python runtime) -- core (running core.p4) -- aggregate (running aggregate.p4) -- gateway1 (running gateway.p4) -- gateway2 (running gateway.p4) -- gateway3 (running gateway.p4) - -###### Nodes (vanilla linux with transparent-security installed into python runtime) -- inet (to core) -- ae (to core) -- Camera1 (to gateway1) -- Game1 (to gateway1) -- NAS1 (to gateway1) -- Camera2 (to gateway2) -- Game2 (to gateway2) -- Camera3 (to gateway3) -- Game3 (to gateway3) - -#### scenario_name=gatweay|aggregate|core -* 1 orchestrator/controller node -* 3 network nodes -* 1 Tofino switch - -###### Switches (with bf-sde-{version} and transparent-security installed into python runtime) -- runs {scenario_name}.p4 - -###### Nodes (vanilla linux with transparent-security installed into python runtime) -- host1 (southbound server node) -- host2 (northbound server node) -- clone (northbound server node) - -#### Accessing the switches and nodes from orchestrator -From the orchestrator node, you can gain access to all other nodes and switch VMs -by name with user 'ubuntu': - -```bash -ssh {switch_name | node_name} -``` - -### Cleanup the simulation environment - -This will remove the VM and other artifacts created when it was deployed. - -```bash -# from transparent-security/automation/p4/tofino directory -terraform destroy -auto-approve -var-file="/path/to/my-tofino.tfvars" -var scenario_name="cleanup" -``` diff --git a/playbooks/env-build/siddhi/env_build.yml b/playbooks/env-build/siddhi/env_build.yml index a53ed116..3ea2f02f 100644 --- a/playbooks/env-build/siddhi/env_build.yml +++ b/playbooks/env-build/siddhi/env_build.yml @@ -14,4 +14,4 @@ # https://github.com/p4lang/tutorials/blob/master/vm/user-bootstrap.sh # Project and script derived in part from the script in the link above --- -- import_playbook: ../../siddhi/setup.yml +- import_playbook: ../../siddhi/setup_siddhi_maven.yml diff --git a/playbooks/siddhi/setup.yml b/playbooks/siddhi/setup_siddhi_maven.yml similarity index 100% rename from playbooks/siddhi/setup.yml rename to playbooks/siddhi/setup_siddhi_maven.yml From a3147a8b394906901d672d02a8a9683c1b1706a3 Mon Sep 17 00:00:00 2001 From: spisarski Date: Mon, 9 Aug 2021 12:53:17 -0600 Subject: [PATCH 19/40] Playbooks necessary to setup Siddhi K8s AE VM image (not currently being used in this branch) --- playbooks/general/setup_docker.yml | 70 ++++++++++++++++++ playbooks/general/setup_minikube.yml | 59 +++++++++++++++ playbooks/general/setup_siddhi_operator.yml | 79 +++++++++++++++++++++ playbooks/siddhi/setup_siddhi_minikube.yml | 19 +++++ 4 files changed, 227 insertions(+) create mode 100644 playbooks/general/setup_docker.yml create mode 100644 playbooks/general/setup_minikube.yml create mode 100644 playbooks/general/setup_siddhi_operator.yml create mode 100644 playbooks/siddhi/setup_siddhi_minikube.yml diff --git a/playbooks/general/setup_docker.yml b/playbooks/general/setup_docker.yml new file mode 100644 index 00000000..c727d36c --- /dev/null +++ b/playbooks/general/setup_docker.yml @@ -0,0 +1,70 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +- hosts: "{{ hosts | default('all') }}" + gather_facts: no + become: yes + tasks: + - name: Remove apt dependecies for Docker + apt: + name: + - docker + - docker-engine + - docker.io + - containerd + - runc + state: absent + + - name: apt update + apt: + update_cache: yes + + - name: Install apt dependencies for Docker + apt: + name: + - apt-transport-https + - ca-certificates + - curl + - gnupg + - lsb-release + register: apt_rc + retries: 3 + delay: 10 + until: apt_rc is not failed + + - name: Add Docker's GPG key + shell: curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg + + - name: Add Docker apt repo + shell: | + echo \ + "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + + - name: apt update + apt: + update_cache: yes + + - name: Install apt dependencies for Docker + apt: + name: + - docker-ce + - docker-ce-cli + - containerd.io + register: apt_rc + retries: 3 + delay: 10 + until: apt_rc is not failed + + - name: Add user {{ ansible_user }} to docker group + command: "usermod -aG docker {{ ansible_user }}" diff --git a/playbooks/general/setup_minikube.yml b/playbooks/general/setup_minikube.yml new file mode 100644 index 00000000..fddc5618 --- /dev/null +++ b/playbooks/general/setup_minikube.yml @@ -0,0 +1,59 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- import_playbook: setup_docker.yml + +- hosts: "{{ hosts | default('all') }}" + gather_facts: no + tasks: + - name: Create minikube binary directory + become: yes + file: + path: /etc/minikube + state: directory + + - name: Get minikube + become: yes + get_url: + url: https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64 + dest: /etc/minikube/minikube-linux-amd64 + + - name: Install minikube + become: yes + command: install minikube-linux-amd64 /usr/bin/minikube + args: + chdir: /etc/minikube + + - name: Install kubectl + become: yes + command: snap install kubectl --classic + + - name: Start minikube to ensure it will work + shell: sudo -u $USER /usr/bin/minikube start --memory=4096 + register: minkube_start_rc + async: 180 + retries: 2 + delay: 5 + until: minkube_start_rc is not failed + + - name: Check Minikube K8s cluster is operational + command: kubectl get po -A + register: kube_pods_out + + - name: Show pods + debug: + var: kube_pods_out + + - name: Stop minikube + shell: sudo -u $USER /usr/bin/minikube stop diff --git a/playbooks/general/setup_siddhi_operator.yml b/playbooks/general/setup_siddhi_operator.yml new file mode 100644 index 00000000..be80e422 --- /dev/null +++ b/playbooks/general/setup_siddhi_operator.yml @@ -0,0 +1,79 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- import_playbook: setup_minikube.yml + +- hosts: "{{ hosts | default('all') }}" + gather_facts: no + tasks: + - name: Start minikube to install siddhi-operator + shell: sudo -u $USER /usr/bin/minikube start --memory=4096 + register: minkube_start_rc + async: 180 + retries: 2 + delay: 5 + until: minkube_start_rc is not failed + + - name: Enable minikube ingress addon + command: sudo -u $USER /usr/bin/minikube addons enable ingress + register: ing_out + async: 60 + retries: 2 + delay: 5 + until: ing_out is not failed + + - name: Create K8s namespace "kafka" + command: kubectl create namespace kafka + + - name: Create Strimzi Kafka ClusterRoles, ClusterRoleBindings & other CRDs + command: kubectl create -f 'https://strimzi.io/install/latest?namespace=kafka' -n kafka + + - name: Install Siddhi K8s and Strimzi Kafka operator (see https://strimzi.io/quickstarts/) + command: "kubectl apply -f {{ item }}" + loop: + - https://github.com/siddhi-io/siddhi-operator/releases/download/v0.2.2/00-prereqs.yaml + - https://github.com/siddhi-io/siddhi-operator/releases/download/v0.2.2/01-siddhi-operator.yaml + - https://strimzi.io/examples/latest/kafka/kafka-persistent-single.yaml -n kafka + + - name: Wait for Strimzi Kafka operator + command: kubectl wait kafka/my-cluster --for=condition=Ready --timeout=300s -n kafka + + - name: Get minikube ip + command: sudo -u $USER /usr/bin/minikube ip + register: ip_out + retries: 2 + delay: 5 + until: ip_out is not failed + + - name: minikube IP + debug: + var: ip_out + + - name: Setup minikube/siddhi IP into /etc/hosts + become: yes + shell: "echo ' {{ ip_out.stdout_lines[0] }} siddhi' >> /etc/hosts" + + - name: Check siddhi-operator is Running + command: kubectl get po + register: kube_pods_out + retries: 10 + delay: 5 + until: kube_pods_out.stdout.find("Running") != -1 + + - name: Show pods + debug: + var: kube_pods_out.stdout_lines + + - name: Stop minikube + shell: sudo -u $USER /usr/bin/minikube stop diff --git a/playbooks/siddhi/setup_siddhi_minikube.yml b/playbooks/siddhi/setup_siddhi_minikube.yml new file mode 100644 index 00000000..3e0926d0 --- /dev/null +++ b/playbooks/siddhi/setup_siddhi_minikube.yml @@ -0,0 +1,19 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Playbook that is responsible for putting together an AWS image for running +# Siddhi with the new "udp" source & P4 Telemetry report mapping extensions +# running in Kubernetes +--- +- import_playbook: ../general/setup_siddhi_operator.yml +- import_playbook: ../general/final_env_setup.yml From 1093a127888ac85a71b8b3f31e28003ddd7a539b Mon Sep 17 00:00:00 2001 From: spisarski Date: Mon, 9 Aug 2021 12:57:59 -0600 Subject: [PATCH 20/40] Sample TPS AE SiddhiProcess K8s CRDs --- .../kubernetes/kafka-trpt-ddos-detection.yaml | 87 +++++++++++++++++++ docs/ae/kubernetes/kafka-trpt-drop-clear.yaml | 84 ++++++++++++++++++ docs/ae/kubernetes/trpt-to-kafka.yaml | 86 ++++++++++++++++++ 3 files changed, 257 insertions(+) create mode 100644 docs/ae/kubernetes/kafka-trpt-ddos-detection.yaml create mode 100644 docs/ae/kubernetes/kafka-trpt-drop-clear.yaml create mode 100644 docs/ae/kubernetes/trpt-to-kafka.yaml diff --git a/docs/ae/kubernetes/kafka-trpt-ddos-detection.yaml b/docs/ae/kubernetes/kafka-trpt-ddos-detection.yaml new file mode 100644 index 00000000..43cbf4bf --- /dev/null +++ b/docs/ae/kubernetes/kafka-trpt-ddos-detection.yaml @@ -0,0 +1,87 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: siddhi.io/v1alpha2 +kind: SiddhiProcess +metadata: + name: kafka-trpt-ddos-detect +spec: + apps: + - script: | + @App:name("KafkaSourcePacketJSON") + @source( + type="kafka", + topic.list="trptPacket", + bootstrap.servers="10.110.95.121:9092", + group.id="test", + threading.option="single.thread", + @map( + type="p4-trpt", + @attributes( + src_mac="intHdr.mdStackHdr.origMac", + ip_ver="ipHdr.version", + dst_ip="ipHdr.dstAddr", + dst_port="protoHdr.dstPort" + ) + ) + ) + define stream trptPktStream ( + src_mac string, + ip_ver int, + dst_ip string, + dst_port long); + + @sink( + type="http", + publisher.url="http://localhost:9998/aggAttack", + method="POST", + headers="trp:headers", + @map(type="json") + ) + define stream attackStream ( + src_mac string, + ip_ver int, + dst_ip string, + dst_port long, + count long + ); + + @info(name = "trptJsonQuery") + from trptPktStream#window.time(1 sec) + select src_mac, ip_ver, dst_ip, dst_port, count(ip_ver) as count + group by src_mac, dst_ip, dst_port + having count == 10 + insert into attackStream; + + container: + image: "spisarski/siddhi" + imagePullPolicy: Always + + persistentVolumeClaim: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: standard + volumeMode: Filesystem + + runner: | + state.persistence: + enabled: true + intervalInMin: 5 + revisionsToKeep: 2 + persistenceStore: io.siddhi.distribution.core.persistence.FileSystemPersistenceStore + config: + location: siddhi-app-persistence diff --git a/docs/ae/kubernetes/kafka-trpt-drop-clear.yaml b/docs/ae/kubernetes/kafka-trpt-drop-clear.yaml new file mode 100644 index 00000000..47788873 --- /dev/null +++ b/docs/ae/kubernetes/kafka-trpt-drop-clear.yaml @@ -0,0 +1,84 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: siddhi.io/v1alpha2 +kind: SiddhiProcess +metadata: + name: kafka-trpt-drop-clear +spec: + apps: + - script: | + @App:name("KafkaSourceDropJSON") + @source( + type="kafka", + topic.list="trptDrop", + bootstrap.servers="kafka-service:9092", + group.id="test", + threading.option="single.thread", + @map( + type="p4-trpt", + @attributes( + timestamp="dropHdr.timestamp", + dropKey="dropHdr.dropKey", + dropCount="dropHdr.dropCount" + ) + ) + ) + define stream trptDropStream ( + timestamp long, + dropKey string, + dropCount long + ); + + @sink( + type="http", + publisher.url="http://localhost:9998/aggAttack", + method="DELETE", + headers="trp:headers", + @map(type="json") + ) + define stream dropAttackStream ( + dropKey string, + dropCount long, + count long + ); + + @info(name = "trptJsonQuery") + from trptDropStream#window.time(35 sec) + select dropKey, dropCount, count(dropCount) as count + group by dropKey, dropCount + having count >= 3 + insert into dropAttackStream; + + container: + image: "spisarski/siddhi" + imagePullPolicy: Always + + persistentVolumeClaim: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: standard + volumeMode: Filesystem + + runner: | + state.persistence: + enabled: true + intervalInMin: 5 + revisionsToKeep: 2 + persistenceStore: io.siddhi.distribution.core.persistence.FileSystemPersistenceStore + config: + location: siddhi-app-persistence diff --git a/docs/ae/kubernetes/trpt-to-kafka.yaml b/docs/ae/kubernetes/trpt-to-kafka.yaml new file mode 100644 index 00000000..134a2b95 --- /dev/null +++ b/docs/ae/kubernetes/trpt-to-kafka.yaml @@ -0,0 +1,86 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: siddhi.io/v1alpha2 +kind: SiddhiProcess +metadata: + name: trpt-to-kafka +spec: + apps: + - script: | + @App:name("UDPSourceTRPT") + @source( + type="udp", + listen.port="556", + @map( + type="p4-trpt", + @attributes( + in_type="telemRptHdr.inType", + full_json="jsonString" + ) + ) + ) + define stream trptUdpStream (in_type int, full_json object); + + @sink( + type="kafka", + topic="trptPacket", + bootstrap.servers="10.110.95.121:9092", + is.binary.message = "false", + @map(type="text") + ) + define stream trptPacket (full_json object); + + @sink( + type="kafka", + topic="trptDrop", + bootstrap.servers="10.110.95.121:9092", + is.binary.message = "false", + @map(type="text") + ) + define stream trptDrop (full_json object); + + @info(name = "TrptPacket") + from trptUdpStream[in_type != 2] + select full_json + insert into trptPacket; + + @info(name = "TrptDrop") + from trptUdpStream[in_type == 2] + select full_json + insert into trptDrop; + + container: + image: "spisarski/siddhi" + imagePullPolicy: Always + +--- +apiVersion: v1 +kind: Service +metadata: + name: trpt-to-kafka-0 +spec: + type: LoadBalancer + clusterIP: 10.96.100.2 + externalIPs: + - 192.168.86.181 + selector: + siddhi.io/instance: trpt-to-kafka-0 + siddhi.io/name: SiddhiProcess + siddhi.io/part-of: siddhi-operator + siddhi.io/version: 0.2.2 + ports: + - port: 556 + targetPort: 556 + protocol: UDP From 4e31abf0933215939e830061ba0d041063e28e43 Mon Sep 17 00:00:00 2001 From: spisarski Date: Mon, 9 Aug 2021 14:37:14 -0600 Subject: [PATCH 21/40] Added k8s instructions to siddhi AE setup instructions Created more links to top-level README.md Fixed "tofino" default variables --- README.md | 26 ++++++++---- automation/p4/tofino/variables.tf | 4 +- docs/ae/SIDDHI_AE_SETUP.md | 69 ++++++++++++++++++++++--------- 3 files changed, 70 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 0b7fc870..062afa89 100644 --- a/README.md +++ b/README.md @@ -32,12 +32,22 @@ We use an [Apache 2.0 License](LICENSE) for Transparent Security. Questions? Just send us an email at [transparent-security@cablelabs.com](mailto:transparent-security@cablelabs.com) or [open an issue](https://github.com/cablelabs/transparent-security/issues). +## The docs +These directories contain other documentation +- [Analytic Engine](./docs/ae) - [How to setup the AE](./docs/ae/SIDDHI_AE_SETUP.md) + - [kubernetes](./docs/ae/kubernetes) - directory containing sample CRDs to deploy the TPS AE on Kubernetes +- [Build automation AMIs](./docs/env_build) - [Creating the required EC2 Images](./docs/env_build/CREATE_AUTOMATION_IMAGES.md) +- [P4 INT](./docs/int_header) - [Description of the P4 INT header added to packets and associated Wireshark plugin](./docs/int_header/README.md) +- [Telemetry Report](./docs/telemetry_report) - [Description of the Telemetry Report UDP Packet](./docs/telemetry_report/telemetry_report.md) +- [Terraform Example Variable File](./docs/terraform) - Example "tfvars" files for configuring a Terraform run +- [P4 Automation](./docs/tofino) - [Instructions on how to execute the P4 automation scripts](./docs/tofino/RUN_CI_AUTOMATION.md) + ## The directories -- automation - contains Terraform scripts for CI and testing on AWS -- bin - miscellaneous scripts mostly used by scripts in automation -- conf - miscellaneous environment configurations -- docs - miscellaneous MD files -- p4 - The P4 source code -- playbooks - The Ansible Playbooks used by automation -- tests - the Python unit test directory -- trans_sec - the project's top-level Python package +- [automation](automation) - contains Terraform scripts for CI and testing on AWS +- [bin](bin) - miscellaneous scripts mostly used by scripts in automation +- [conf](conf) - miscellaneous environment configurations +- [docs](docs) - miscellaneous MD files +- [p4](p4) - The P4 source code +- [playbooks](playbooks) - The Ansible Playbooks used by automation +- [tests](tests) - the Python unit test directory +- [trans_sec](trans_sec) - the project's top-level Python package diff --git a/automation/p4/tofino/variables.tf b/automation/p4/tofino/variables.tf index 56ead961..aed573a5 100644 --- a/automation/p4/tofino/variables.tf +++ b/automation/p4/tofino/variables.tf @@ -30,8 +30,8 @@ variable "availability_zone" {default = "us-west-2a"} variable "tofino" { default = { - sde_version = {default = "9.2.0"} - bfrt_ami = {default = "ami-090b409398f016b84"} + sde_version = "9.2.0" + bfrt_ami = "ami-090b409398f016b84" } } diff --git a/docs/ae/SIDDHI_AE_SETUP.md b/docs/ae/SIDDHI_AE_SETUP.md index 64262098..5cc33a88 100644 --- a/docs/ae/SIDDHI_AE_SETUP.md +++ b/docs/ae/SIDDHI_AE_SETUP.md @@ -1,10 +1,10 @@ # TPS Siddhi AE The Transparent Security TPS AE is an implementation of transparent security -analytics engine based on Siddhi. It is able to identify a DDoS attack in about -1 second by examining telemetry reports generated by a switch or router. This -will also identify when a mitigated attack has ended based on drop telemetry reports -from the SDN controller. +analytics engine based on Siddhi. It is able to identify a DDoS attack in less +than 1 second by examining telemetry reports generated by a switch or router. +This will also identify when a mitigated attack has ended based on drop +telemetry reports from the SDN controller. The directions below are how to run this as a simple Java program (as with our "lab_trial" scenario tests), but these scripts can also be deployed to any @@ -24,16 +24,18 @@ Kubernetes where the siddhi-operator has been installed. There are several methods that can be used to run Siddhi scripts. ### Java application +The top-level Ansible playbook performing the actions listed below can be found +[here](../../playbooks/siddhi/setup_siddhi_maven.yml) -- Install Java 8+ (CI is running OpenJDK 11.0.11) -- Install Git client (apt install git) -- Install Maven client (apt install maven) -- Install and start Kafka to port 9092 ( - see https://www.digitalocean.com/community/tutorials/how-to-install-apache-kafka-on-ubuntu-20-04) -- Clone the following Git - project: https://github.com/cablelabs/siddhi-map-p4-trpt.git -- Open shell to the cloned directory (e.g. cd ~/siddhi-map-p4-trpt) -- Execute +- Install Java 8+ - [playbook](../../playbooks/siddhi/setup_jdk.yml) (setting up OpenJDK 11.0.11) +- Install and start Kafka to port 9092 - [playbook](../../playbooks/siddhi/setup_kafka.yml) + ([installed based on these instructions](https://www.digitalocean.com/community/tutorials/how-to-install-apache-kafka-on-ubuntu-20-04) +- Install Siddhi for Maven - [playbook](../../playbooks/siddhi/setup_siddhi_p4.yml) + - Install Git client (apt install git) + - Install Maven client (apt install maven) + - Clone the - [siddhi-map-p4-trpt](https://github.com/cablelabs/siddhi-map-p4-trpt.git) Git project + +Execute following shell commands ```bash cd {siddhi-map-p4-trpt directory} @@ -41,14 +43,43 @@ mvn exec:java -Dexec.mainClass=io.siddhi.extension.map.p4.StartSiddhiRuntime \ -Dexec.args="{siddhi-script-location} {siddhi-script-location} ..." -f pom.xml ``` +Open shell to the cloned directory (e.g. cd ~/siddhi-map-p4-trpt) + +Execute following shell commands + +```bash +cd {siddhi-map-p4-trpt directory} +mvn clean compile test +mvn exec:java -Dexec.mainClass=io.siddhi.extension.map.p4.StartSiddhiRuntime \ +-Dexec.args="{siddhi-script-location} {siddhi-script-location} ..." -f pom.xml +``` + ### Kubernetes Other than a Siddhi runtime server or Java application, another means to deploy Siddhi scripts is on Kubernetes with the [siddhi-operator](https://github.com/siddhi-io/siddhi-operator). -- Install siddhi-operator -- Apply CRDs - - kafka-operator.yml - - parse-trpt-udp.yml - - eval-pkt-trpt.yml - - eval-drop-trpt.yml +#### Setup +The top-level Ansible playbook performing the actions listed below can be found +[here](../../playbooks/siddhi/setup_siddhi_minikube.yml) + +- Setup Docker - [playbook](../../playbooks/general/setup_docker.yml) +- Setup Minikube - [playbook](../../playbooks/general/setup_minikube.yml) + - Start Minikube - [playbook](../../playbooks/general/setup_siddhi_operator.yml) + - Enable ingress addon (/usr/bin/minikube addons enable ingress) + - Install siddhi-operator & Strimzi Kafka server operator +```bash +kubectl create namespace kafka +kubectl create -f 'https://strimzi.io/install/latest?namespace=kafka' -n kafka +kubectl apply -f https://github.com/siddhi-io/siddhi-operator/releases/download/v0.2.2/00-prereqs.yaml +kubectl apply -f https://github.com/siddhi-io/siddhi-operator/releases/download/v0.2.2/01-siddhi-operator.yaml +kubectl apply -f https://strimzi.io/examples/latest/kafka/kafka-persistent-single.yaml -n kafka +``` + +#### Deploy AE +To deploy the AE onto minikube, please look at the following example +SiddhiProcess CRDs that can be simply deployed with the standard "kubectl apply -f" +command: +- [UDP to Telemetry Report JSON to Kafka](./kubernetes/trpt-to-kafka.yaml) +- [Telemetry Report JSON to DDoS Detect](./kubernetes/kafka-trpt-ddos-detection.yaml) +- [Telemetry Report JSON to Drop Clear Attack](./kubernetes/kafka-trpt-drop-clear.yaml) From 48de3a5b1945987b19e5a14848a180bae59b0473 Mon Sep 17 00:00:00 2001 From: spisarski Date: Mon, 9 Aug 2021 14:54:31 -0600 Subject: [PATCH 22/40] Removed redundant example --- docs/ae/SIDDHI_AE_SETUP.md | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/docs/ae/SIDDHI_AE_SETUP.md b/docs/ae/SIDDHI_AE_SETUP.md index 5cc33a88..4352c734 100644 --- a/docs/ae/SIDDHI_AE_SETUP.md +++ b/docs/ae/SIDDHI_AE_SETUP.md @@ -37,16 +37,6 @@ The top-level Ansible playbook performing the actions listed below can be found Execute following shell commands -```bash -cd {siddhi-map-p4-trpt directory} -mvn exec:java -Dexec.mainClass=io.siddhi.extension.map.p4.StartSiddhiRuntime \ --Dexec.args="{siddhi-script-location} {siddhi-script-location} ..." -f pom.xml -``` - -Open shell to the cloned directory (e.g. cd ~/siddhi-map-p4-trpt) - -Execute following shell commands - ```bash cd {siddhi-map-p4-trpt directory} mvn clean compile test From 618d18e9269205432aa0d02ac67244c9f38c6c18 Mon Sep 17 00:00:00 2001 From: spisarski Date: Mon, 9 Aug 2021 15:00:45 -0600 Subject: [PATCH 23/40] Added in networking config to the tofino-int.example.tfvars automation example as this will break in somebody else's environment. --- docs/terraform/tofino-int.example.tfvars | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/terraform/tofino-int.example.tfvars b/docs/terraform/tofino-int.example.tfvars index e80362bd..692aaaac 100644 --- a/docs/terraform/tofino-int.example.tfvars +++ b/docs/terraform/tofino-int.example.tfvars @@ -3,3 +3,5 @@ tofino = { bfrt_ami = "ami-xxx" } siddhi_ae_ami = "ami-yyy" +vpc_id = "vpc-xxx" +vpc_subnet_prfx = "xxx.xx" From a661bda6b69833bea9e0908e3433e4211383c562 Mon Sep 17 00:00:00 2001 From: spisarski Date: Tue, 10 Aug 2021 10:18:16 -0600 Subject: [PATCH 24/40] Changes per code review. --- docs/ae/SIDDHI_AE_SETUP.md | 37 ++++++++++++------- .../kubernetes/kafka-trpt-ddos-detection.yaml | 5 +++ docs/ae/kubernetes/kafka-trpt-drop-clear.yaml | 4 ++ docs/ae/kubernetes/trpt-to-kafka.yaml | 5 +++ playbooks/general/setup_docker.yml | 3 ++ playbooks/general/setup_minikube.yml | 2 + playbooks/general/setup_siddhi_operator.yml | 6 +++ 7 files changed, 48 insertions(+), 14 deletions(-) diff --git a/docs/ae/SIDDHI_AE_SETUP.md b/docs/ae/SIDDHI_AE_SETUP.md index 4352c734..364d1350 100644 --- a/docs/ae/SIDDHI_AE_SETUP.md +++ b/docs/ae/SIDDHI_AE_SETUP.md @@ -1,16 +1,20 @@ # TPS Siddhi AE -The Transparent Security TPS AE is an implementation of transparent security -analytics engine based on Siddhi. It is able to identify a DDoS attack in less +The Transparent Security Analytics Engine (TPS AE) is an analytics engine +implementation based on [Siddhi](https://siddhi.io/) event stream processing +(ESP) and complex event processing (CEP) framework. The Siddhi scripts +developed are able to identify a DDoS attack in less than 1 second by examining telemetry reports generated by a switch or router. This will also identify when a mitigated attack has ended based on drop telemetry reports from the SDN controller. The directions below are how to run this as a simple Java program (as with our "lab_trial" scenario tests), but these scripts can also be deployed to any -Siddhi compliant engine containing our UDP & P4-TRPT extensions (and a running -Kafka instance). Additionally, Siddhi scripts can also be converted to run in -Kubernetes where the siddhi-operator has been installed. +Siddhi compliant engine containing our +[UDP](https://github.com/cablelabs/siddhi-io-udp) & +[P4-TRPT](https://github.com/cablelabs/siddhi-map-p4-trpt) extensions +(and a running Kafka instance). Additionally, Siddhi scripts can also be +converted to run in Kubernetes where the siddhi-operator has been installed. ## Host Software Requirements @@ -29,7 +33,7 @@ The top-level Ansible playbook performing the actions listed below can be found - Install Java 8+ - [playbook](../../playbooks/siddhi/setup_jdk.yml) (setting up OpenJDK 11.0.11) - Install and start Kafka to port 9092 - [playbook](../../playbooks/siddhi/setup_kafka.yml) - ([installed based on these instructions](https://www.digitalocean.com/community/tutorials/how-to-install-apache-kafka-on-ubuntu-20-04) + [installed based on these instructions](https://www.digitalocean.com/community/tutorials/how-to-install-apache-kafka-on-ubuntu-20-04) - Install Siddhi for Maven - [playbook](../../playbooks/siddhi/setup_siddhi_p4.yml) - Install Git client (apt install git) - Install Maven client (apt install maven) @@ -50,24 +54,29 @@ Siddhi scripts is on Kubernetes with the [siddhi-operator](https://github.com/siddhi-io/siddhi-operator). #### Setup -The top-level Ansible playbook performing the actions listed below can be found -[here](../../playbooks/siddhi/setup_siddhi_minikube.yml) +This Ansible [playbook](../../playbooks/siddhi/setup_siddhi_minikube.yml) is an +example of deploying the operators below onto a Minikube instance. -- Setup Docker - [playbook](../../playbooks/general/setup_docker.yml) -- Setup Minikube - [playbook](../../playbooks/general/setup_minikube.yml) - - Start Minikube - [playbook](../../playbooks/general/setup_siddhi_operator.yml) - - Enable ingress addon (/usr/bin/minikube addons enable ingress) - - Install siddhi-operator & Strimzi Kafka server operator +- Obtain access to a Kubernetes cluster + For testing on a single machine, Please follow the instructions below: + - Setup Docker - [playbook](../../playbooks/general/setup_docker.yml) + - Setup Minikube - [playbook](../../playbooks/general/setup_minikube.yml) + - Start Minikube - [playbook](../../playbooks/general/setup_siddhi_operator.yml) + - Enable ingress addon (/usr/bin/minikube addons enable ingress) +- Install Kafka ```bash kubectl create namespace kafka kubectl create -f 'https://strimzi.io/install/latest?namespace=kafka' -n kafka +``` +- Install siddhi-operator +```bash kubectl apply -f https://github.com/siddhi-io/siddhi-operator/releases/download/v0.2.2/00-prereqs.yaml kubectl apply -f https://github.com/siddhi-io/siddhi-operator/releases/download/v0.2.2/01-siddhi-operator.yaml kubectl apply -f https://strimzi.io/examples/latest/kafka/kafka-persistent-single.yaml -n kafka ``` #### Deploy AE -To deploy the AE onto minikube, please look at the following example +To deploy the AE onto Kubernetes, please look at the following example SiddhiProcess CRDs that can be simply deployed with the standard "kubectl apply -f" command: - [UDP to Telemetry Report JSON to Kafka](./kubernetes/trpt-to-kafka.yaml) diff --git a/docs/ae/kubernetes/kafka-trpt-ddos-detection.yaml b/docs/ae/kubernetes/kafka-trpt-ddos-detection.yaml index 43cbf4bf..5f06aa05 100644 --- a/docs/ae/kubernetes/kafka-trpt-ddos-detection.yaml +++ b/docs/ae/kubernetes/kafka-trpt-ddos-detection.yaml @@ -11,6 +11,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +# This is an example of a TPS-AE SiddhiProcess CRD for reading in Packet +# Telemetry Reports as a JSON string from Kafka and sending out an HTTP POST +# when 10 packets in a 1 second window with the same originating MAC, +# destination IP, and destination port. + --- apiVersion: siddhi.io/v1alpha2 kind: SiddhiProcess diff --git a/docs/ae/kubernetes/kafka-trpt-drop-clear.yaml b/docs/ae/kubernetes/kafka-trpt-drop-clear.yaml index 47788873..d3d970c5 100644 --- a/docs/ae/kubernetes/kafka-trpt-drop-clear.yaml +++ b/docs/ae/kubernetes/kafka-trpt-drop-clear.yaml @@ -11,6 +11,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +# This is an example of a TPS-AE SiddhiProcess CRD for reading in Drop +# Telemetry Reports as a JSON string from Kafka and sending out an HTTP DELETE +# when 3 reports with the same hash and count arrive. + --- apiVersion: siddhi.io/v1alpha2 kind: SiddhiProcess diff --git a/docs/ae/kubernetes/trpt-to-kafka.yaml b/docs/ae/kubernetes/trpt-to-kafka.yaml index 134a2b95..46b43480 100644 --- a/docs/ae/kubernetes/trpt-to-kafka.yaml +++ b/docs/ae/kubernetes/trpt-to-kafka.yaml @@ -11,6 +11,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +# This is an example of a TPS-AE SiddhiProcess CRD for reading in Telemetry +# Report UDP packets, determining the type, sending the JSON String to the +# Kafka topic "trptPacket" for Packet Telemetry Reports created on a P4 switch +# and topic "trptDrop" for Drop Telemetry Reports. + --- apiVersion: siddhi.io/v1alpha2 kind: SiddhiProcess diff --git a/playbooks/general/setup_docker.yml b/playbooks/general/setup_docker.yml index c727d36c..a171583c 100644 --- a/playbooks/general/setup_docker.yml +++ b/playbooks/general/setup_docker.yml @@ -10,6 +10,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +# This playbook is capable of setting up Docker on deb Linux machines. + --- - hosts: "{{ hosts | default('all') }}" gather_facts: no diff --git a/playbooks/general/setup_minikube.yml b/playbooks/general/setup_minikube.yml index fddc5618..d5f60463 100644 --- a/playbooks/general/setup_minikube.yml +++ b/playbooks/general/setup_minikube.yml @@ -11,6 +11,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# This playbook is capable of setting Minikube and depends on setup_docker.yml. + --- - import_playbook: setup_docker.yml diff --git a/playbooks/general/setup_siddhi_operator.yml b/playbooks/general/setup_siddhi_operator.yml index be80e422..665fee63 100644 --- a/playbooks/general/setup_siddhi_operator.yml +++ b/playbooks/general/setup_siddhi_operator.yml @@ -11,6 +11,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +# This playbook is responsible for starting minikube, adding the ingress addon, +# setting up Kafka as well as the siddhi-operator which makes it possible to +# deploy CRD's of kind "SiddhiProcess" to K8s. +# Please note that Minikube is not required to install the either of the +# operators (Strimzi Kafka or siddhi-operator) + --- - import_playbook: setup_minikube.yml From 60a9646205976833a587c62e83537c9c086c2f3b Mon Sep 17 00:00:00 2001 From: spisarski Date: Wed, 11 Aug 2021 12:28:53 -0600 Subject: [PATCH 25/40] Added some screenshots of the Siddhi scripts as rendered from WSO2 Streaming Integrator Tooling --- docs/ae/SIDDHI_AE_SETUP.md | 13 +++++++++++++ docs/ae/kafka-src-drop.png | Bin 0 -> 50842 bytes docs/ae/kafka-src-pkt.png | Bin 0 -> 49479 bytes docs/ae/udp-src-trpt.png | Bin 0 -> 55991 bytes 4 files changed, 13 insertions(+) create mode 100644 docs/ae/kafka-src-drop.png create mode 100644 docs/ae/kafka-src-pkt.png create mode 100644 docs/ae/udp-src-trpt.png diff --git a/docs/ae/SIDDHI_AE_SETUP.md b/docs/ae/SIDDHI_AE_SETUP.md index 364d1350..0700da0a 100644 --- a/docs/ae/SIDDHI_AE_SETUP.md +++ b/docs/ae/SIDDHI_AE_SETUP.md @@ -16,6 +16,19 @@ Siddhi compliant engine containing our (and a running Kafka instance). Additionally, Siddhi scripts can also be converted to run in Kubernetes where the siddhi-operator has been installed. +See diagrams of the Siddhi scripts as generated by the WSO2 Streaming +Integrator Tooling: + +![image](udp-src-trpt.png) +UDP Packet Telemetry Report Source to Kafka JSON + +![image](kafka-src-pkt.png) +DDoS detect + +![image](kafka-src-drop.png) +DDoS drop + + ## Host Software Requirements 1. Java 8+ diff --git a/docs/ae/kafka-src-drop.png b/docs/ae/kafka-src-drop.png new file mode 100644 index 0000000000000000000000000000000000000000..7f492b89ab1b4d60c9284f24137141180f71a146 GIT binary patch literal 50842 zcmb5W1yq#l7e0!5FhImW5EL*#8Vo`}Kt;d-34s9_%Ai|P8VgjUOFAW`yNx5=Fm#FJ z457f#arbu^)c?J|yY9NI<#LqyX5QHE-p_vav)?xga#9pzbYvtXBowGSw-rf9NOed^ z_Ldyn5C2oafPMgf{bO|trF;;6oDS-|hJVvqOWe0sGS{=VduXXkqHktys>@~d*iu*5 z%*w#rdTw8(C<)0~64dRR%18*qG$3GMvCB1czTjF-{ za{^g%GVeX*lGkSszFyh)`8EA9jxo$Js*?#nKG00vB4c{CC|W7z>-QeD$3OIFMRa~n zM2~*6=$zo8Wo`!B!N+%GWUi7Y4{K;>WEgA}nD$X=u;fLg?twwVPlm>ThhE+1L5ZDz zl6<20%qiGgx_?+&S6?4rzC768nXb-^N>?xJd_gJ7%EH1Ttcpwhg_Jv!rq;azoEv46)(Sd_kGOp@bG-bhx=9&O+nGo(bSyx z&%XE^6BtM(8+t|JVgPEUH=kp9!F9ft#nQ@3DO0PwZsyF7DzEk*K4KYRe8$0jf;%5b z()~uQORE04Qj*+MORTuV@UsItmrgOEqN1W4)~1pOTkCUu#X=ghi&O1&b#*y*w8U|| zQck7MQGK*JKZYqZi;Rej92*@?`2N)+&32|&$YC*g(0+oxd}YKlxY%yUxp2HWI`h+~ z8>qI4Xjg6+r|?`^?x(PDep5wdW#w2a(*65Wjk_{}muYEfFe7 zEVGJLy~Z#;nNr&s{}(S_@SAkU&|cM>9IXv-Sf43Gr(^g`2TIUBqHD==(W06LG&9dh z$vevIt!yS+Zu;lh&S0FkRwpbiE&b!;K_}vn=j7~MKUs<{*jo8P35F#nDk_Q_D7A~_r`cJ!PhJ^! z_w~q&uHLTprf)xOUUm+)Ir7?*Td)CG7zY&{UFr!IIaV<-?ITBy2w9H4K0{BR|Kx`x zVQxsK|@3Ht*XjsuFC&(Hvhc{x1Ek8(lpJ6-T;aT6Ap{cG}YKvVSa^*o8btL;8 zVHD*=^%gyeXKgww`<|8Oh^LKC+_A8ob zhN#0(_ER#L6bHMxk=WonWwW!hRVG-NnNu+NM#0OC;R0Qsp6!!y7FF*lcPw;Sn_}TJ zY`YDct~ylV926Rwa*{)Zm5Zx!aE$1ePQK{Tu%#pjxo(xg582G?!<3X=z4^w>DAV4& zR&RRYj;x2`aDZj@9XQg_rR|!&xM&?I>WqOg<&_lF>ab_?Ou!@$lh8%dGE3S=!pFNJ_dhr5Ls+k{vwQ0$YoX6+a+x`*t!a;NrKa_WJs??jPIMBmW+H z>>hz@tH~bdXmuAx70mJ4f5nM~`i{3)>OlmHSMyp_bAB-qZx1O5MxU*s2g3sLC zoG{fcm8P0mFdazv`7_JjbTNnYzs&J6;VDmu z5^=0+Z&&j6_BI+Q$p>eh`2PK=e6xXGTyI51g~aUn%@I!?>xoFay4u>>oU4V&3pCXa zE=|_5L@PTul-N#p#|jKS6^8@mfJ;e|-Y_u~6cWOeOG*4%TGt%jWIX3_PJ zYcLrG5r;rE*{(+mc%}E@(fW&=e)YAs`@NGu?fUr(z{e zF5+5j0E@h|V#;%6N=nKPNl7%P>KhyL%i!d9f_o!BGnDo41^6ga!Wr;oing`|;9js` z(~Y55rRFNu&ED8_e`$&o&Ih*u+e=k!+*MyM1792T`9TYu2C_|)ojH*Rgk2%mVV?39 zo^1gq6Br%MgA#JtD5_YW6}Fih9Js=A{kled2v6ZZInSOwlfH9DK}#zNRvC~nYkPGogu z_+i=FAx#UOJmDO}iA+zJg#u#GsG>)-1NI0p9ZS=g$tmzR}2_ z9d&dPMn*<>jXGFqy(GuR$Bk-y&jkw!TTjMhKdv?c|L!pLA>?DmF>vWe=!FcrvmRR7 z*aT%}W}d%rfvftofZ0bF1rpWZ#0Ep4igLiQ$!Pa!l{*+NP4|$2Q+R**F|YNc0+&XS zLcFAx|K|t35R#%?w^kK3{QDg2-19AU%!e2G~t|hyR!stZ<~FCgY^pRj@>{3 zGUCkiI6`k^BWG=$H%9mmBPESSwi4f({&G;b+(ez4XeT6ZAMl^3A%YFJ^rOFDdi{CH z)snOOZWE6iyHA>YzI&Dki%9r?yblc2nWJ+%4v}5!mwrx8PF?Krori{Tf}P3tz7Fz{ zFxNFVSB^Aa6BZ5;XGVVMR)_0=7`+@>Xfs-K%--SYB;l2LcPSG|kZ*kXF63&WH13>? z2b9z|SFTTZ`lZ;)L9gLdz5!NI{s`j}^u!%``L2a#Z*+KpDW82EdGIguiY@6OCqmFT z&;6%pyq1@<Ce`jIDUK-UIQrU8MVwn*ZFMYU?dOYSY^JL#McJ|_I!#6P(4j-U1vZ{q4o^~NOB~(y_gnc4 z;Bu25%3qHK;2`5~xiC*fb2=^BRVy<|(yL?moVbm8bChjHmdQ+CscHa=oB}`|HhVlC#U1q?<;*#KGuH~$Nei zFRyw_bUodD&gc^KH(=kC`wC{LG)fD&coy7m6D=k~#?a6(?3zo&#^xrWqrF)}$(Qc* z=`;w5eSGh{JU!d}3*-2n%v0r?wO3u49UJ=q045oo)=4K~ln};ep0S_ebZ@$x;}ct3 zzfup6G3Ow5bVtlXahu7bv2T7<*ESDnb72bd9>3D9HsQx(1#@j2n_Z{ApA+Z&F*rDw zdN1A`Q)rsjdcoZo>>_zv(@v(Od-loryD2hq%C4V&@nj!ql;hod3rv$P%X@XEp5(Bu zj;pHZGRj%u^|QI%#|YAezCv%Xmj zPv{yq{E|>xGBqB~jACP}#S}k!#lNwZHu7U+bapmlje^dgrF=QXtlWWFDLE*W{?(D}7!=De>=x>)Y zp}_tzORb8g133~j+!_k`sc0MHq^1)6mY7>Z0ax{#40_r=^cFuZ1-Pi3tHMQ?a`Y&% znbR6^X7lQET+CUZzaYu5~_r?Sb1x+0eKX)js&N?TpT7*DW?XJG*>* z%S}p=#pv^u+N$-91DX)R--?~iY$>rxZ<;!KEEe`heQp&D8wu4->+=iEgjM}$qn|CYbRy%10BZ1~wI>{wX9{h8e!A%ueeIsV zwQ-VMNxl+G`TA1d+CoRlElPCGcoL&)$CqtK%@Zv0_f}*DP$j3Tyy-+Wyux`~yY-`$ zZaj!-Yj0cZGe1q}@7Cwt91Beyin*DMhvcV`?BL;!2HLBO(+kb&6sudsl19>Rt3)^R zQrA_wx+kJky~6lx_(ioiDmHG#yHj>8hF%3E6?r@}h1KNCCmY--TY2$Qzl-k}lX|SQ%f~f)L#%9U&>_jJTrNK-NJ;mr&ERqP2^MWJvHEq` zZklzUgZ_}1$AI^le0t2PufRCzb0b?$4|^CH}bKYc}y?O z^qH#F*?iDFw0Wn)?UHfhqW%+Kr3^mf7x*^b zRAUP?9S1D&=|(-NL6A14H_1y&`zIx-zQH>$&Yms&TOrPHa9k++g1~9S)Dj>LWe(`YbbuD=oZxX0^Ge_Y30>rv0}^svHi#&YSj?pdnqsENt49%xxWb8zH>Ua_sob6wE+>pXl685Q8s`4$_BfmB`Yg>&swl~rD z5x$1;g^A8c(4INd0ZAEQrb|BG*yH|q;)g_|&9!LW$K#x7!nAXx+b#nF1~E%>eMzuL z!CSX@+Am+pX>pd8MYRSuhA+-@WKxCA!65n^w`5Q#nthO-S119;bmNpG%3z=*6`h-C zYc`b8=de_ky2h=~D@1KH@gRLGt=!xXvRZ=-kEEm$!elH(q_w%#kYFP30f{<+w3;)0~X_~MjN zUqNIehK7lmc`A3TJ}5XiQEdDA2f4^ngSn^yp)A)*Y*gWarMa%!6D-zBxq6T_wD}mj zDnNb+p{8TAppV0XRi=IibXw3~bL7_oMy;hmRS(}0oy5EOjRsw4rkQqgh}naZzAZI~ z!{))ZC^HL-dR2Q{n@(qt$cEtzr7@?-<89h1Eg;w8GgUG>-6<+@i_xx_wKhF{{WPu3 zHUEUo^(8}GARDQu9XLUU>0j04@A%AH6&uy_lwl}gqq9Bjr+2-QYm_2nO1O?g`D>xBLsmXxgxD7z zRE&(7&-Q(s%WJ5))3v%VA>*w5HXtB?OQ4XQxl_AyIbAMNVRNmX$3LN2Y+X6pS@CD= z{jX6^Zx!}C7#s4$`}XFG^DeelBY&(HC-0?i|B`BSuy+URnks(sgKN`)o}L~^KkX5R z;^JbDsMkj{Swf8*5zVAIgd29+B(+>AOtrZ~=(YJ)%SjIifBc}AnC za=e$ZlXb08z@}@!4qtJAK>@rrSLuOmXVq;EA-lVG-^C3%DFL1#U|m(yY{L0p?M;Pz z(aYK4U0K>we8oDW0wgP=v)|95FbGN|ba_70gaj|&;@8ODXf9l=PTM!%bhX@Zi0{Q_ zrR7N%y&02+hkJ3SXj34Wuz&iL>U1y=d~94IGjB5rnWhYO0@}JM!6qU#u8f$7=PLoqlrd*cCXOteO85*iouB_9Hph+rENNLw7I6MxA!%PZ6Kn)D@j~j9wQX8cM7hrj5i@qHR5v6uB z?3OOTzS+!eF5{CJyVR;~ZTDi;BQ`di=!I?Xo;!CAL48RfQKv*T;E1_oi-mLn$AjWC z#?el&Ko%HpnyoF;rm)z8T0cCiGVR&%Qt0*XpM$MROG`N}pKCSZ8sBh&aDee18yn-`5y))O z*mffkymoEUM{Fz0VQ_nM&TedCEY04Nnha;;yC#4^r7vZkHMb zm6fF?gPUURbmd!?y~4p#y|#*sG>p%PdVNy_xSEUCx61|{CNzN|p%U7H{Z7>^EztF% z?<+K}Pm~Q4bd>Wmc6j=f{lW!tU|%dPEEM`|X9$C(a|XlZ%e6f{Jve=S^WKF1ehoef z^4tgx;q^AxTyjlu&*1cqL{s6$$gAqkbCWD;t5)}^g$6AV?0%AwQS~Bzz_LCag15t~ zM)!j{_Y6-G#Gt+d**e}cJ|4B7mTXg`xu-`hAgL^-^@5baYL2eM}4YUY} zh~uTB*Rkfsl+OV*gvAAMEcO@uBJ~~{ADtXvHcwNH9z-}*py0-4X5xKMa_>hP!#g&@YRviMvEac3LT$>6%SmfASBx8%#G8ZdQ`;lXEj9!@feYl@r zY;@Ol0yx*o?m6?Xl3t?_#7)%;lU3(LNxfmt8tcn5?IJJst94Fx6nh5qs3#8D%__rI ze#|KT-lg4_Uo51rTkP&D@*IOMMKDu)bxM+ zNaxXGtrW_P3=Q7N7>C!!@DoW-PFytl_LheL`JRkFKvu}2y=`Z^GyeVe_RLVF zcCJ=b?7jm__3iC4Dcc5C7R*96Q*nS$(r%Q#i~?4xc~;XwzxMdIfB$9W=T{kW+02h} z9;f9q>c7vH!p59&8k?VXUdXoNHA{5Iyl%bqsE=4+M8rqP(I$};<^8=22z}yWw>7YD zx%HtB2gEg=@@r@*pELKs6*ol)#v|N}2cI#oQA0avQ)?kmfD44KwIzIJkgsnHa5d?x zTaE~_f#J(IQ!>yly@yDMJjH(TqGV`jsF2g7Z(_wIiP_m%TxYpQf+rVUl>MyQW{ykw zcw=O|n0%z7ib^o>wi(5_`UqRVx}50|9mK>VpbB*d@M-tJWqNP?@(3yJ?Uc-&2Gu;)MhbEe)fGuB}?K9_d7(K{mNS7$Q zHlT)EXVRQbT3-qHnr}Wg$g$QRbO$BZ3G4~E*J&?FLUDX_386^#CZACa7K@J3(lY8O z%HBUQ7OxV z&Qe28n?YyZNe)=IH-=ra&$*JDLskCNFDEzG(^&zbyMG?LfW3T0amaN&y<%&PL$k=@ z=0zzfEZGs2F0=nw6SC=x>53oTj+Y6^*N3=%4AF{6efjTgbc&(`lwVtnGq)ap0Rg#R%*nc5q5}BVgVa7$xPWVF4Bsvh-57TEqF)G9R8$(9 zn@Ca@w&~Q|x4*_(&CBbwKx^89b{7ky>&IWsVo9T;I_OL-O+a^Pa2CrtglV;-x5KpH z4{+mx&YK%hdn9j+T<$%&e73x8{UVBzk~XgU>gBMd5#?_Yf)|jA!RyIn4=7zJ5?E)k zr9MFIC!W-HNrB~LXAw}n`RpdHYfg?({KNa!La8l2vC8YTF}5x!Y#FB!DeS1Cm#en5 zXAc5BfQCb}-lilYYr_NX@N5=q5BH>DLpo2!{|}2LXB4Ek_7SFFVCA7 zOH|HnY^I8u%j)U5A7K>w2v96i$iB@;e#qOeFgYWr;CS1&tOCgKruvI>U)k0HajzBa zx|WivoO0gP)xFp`WLFE|`fA9@Dio_peR%x5&?)nw3QexeVQp$Nst?6H{iSi5Gv&3- z&4pzQLe|Mx!PeH+;I6&`QzeVY77m`HNW38#X54?|R7~sRs$|)|f+g+`ZP#q(6GSz) zp7>QqMnv#V*M4VqA3(gPfc9o#%?|)1T`yeoucjQlb>Qwpk7+M6JEKH??HUD)OT|XS z8Ej`eI@U~L+nD~p4<}I=blTjI&R@8eEJ<-%S;@U2kF)6es&F`OLe)3-E<=00blJPu z)R$Bg6shcX@v$%E_w78Vo5}m|Ba`5%jLMC-H>ktx*l)UAjBgqrP4V<8jZ1Z4UAuce zmUHK=(<6$_8sdkZ;d^rRO_lHUobxUJyc+tB4xJV@wwapC)@yxET>M2UF1~w#4)e9z z8~YzHC+QvnDmdHjP>?q;XOILSpcIfTq=IY~*F{7$At6zG;ZEVHIpJD4mnWoEYWq5M z=n6dppQ$2bnwJ61yPadWvU4PYhu-1xByIQq1DWIdPx!yZyC&wF>9RB`27D&O# zPg|R$RadWES!`{iL{6_4oq%6%ZY~oW+vR)V0;X*h<&GaS?5q~X$Fvsm&-#`>GnH;? zNgO?lYP+$`j#ChW0<8z;4XQi>i|3~cmjsPr^F6QAtr6GODTY%t}T0Q43NXXKy^xlaDfVm{ES-Pk&}nFbda!4G0n%8;hrlxMT3l8T;Q=ttT!8tR%QwgK^v zzr7RX&_k=T_J5T)*L0D-xz6u4t9qdB6B;JeJ+uf}9`nIvi`EVIn+{V?%XAB;jgyX# z>R(~+|N4zPA4c~1!5p!yqwxIGr?Z4gyqWV>Pf}f7Lm<>uO04mTC#t=Tw=2r)nhFNv zI{Nz58^a0}B=y|+x0k{bJReTeZ?3C$Us6eB644p(+TWN=6LVBQKzCY^RYtz2lFPF+ z21Y`)tZLybXcvF89lu6XxIm$9~*~ z3M7QzV||h((p26~o=U0MTuaf~+<#K^T@b7%$FaKLAX>uSQT)><%O*~8QKD}}%y;W& z78ecztr=-Izy(#=`u70=O3#)c1|T&mPXm4`#CcO zG;3HxtaiSFg!wX^7x4J(KW#+B8p``^YbvO4s#-E3llLXP2j8ODd85Ge3Lp9NmS zZL{33g>%ME&vrE2iY8c<6!Z_w2+h_K=c3FsRv=ywm(2NadL{msbbWK7Msr!fg!M*d zWZH|pW%=L}@&0YRTLxv4ovDeejCS=Ceds4>b$L}&x#AUz3~td`Qg0bjb*ek;N2h~3 zqKVbd8?_DW)i-RhvY*HD^U$3xk(F0oX8SLy-1TppU%-B=){Hko2QXou@Tu+X97qWb z{$;b{F}{dskmb>z0v8FsXzbw?>Qg**W0ZTW?21GM6!mp`GJP)Z?C=BG58da~L%-ff z(J_uB1?lrpP5z?8;Y{qy1qC(5jaLkm(~T_cEEJN%1F$UwY1H9EChA zEVfHK%OCS7h3XlyHqxi7r2R`<70$j3;x0I6arIQ789#?8i##}!le6FBV)>jhR-Jwc z+6NC5`cut*w`?@-eerCtaA!2h4>FpEB(^CJ#-XK(M>7RcRxX($MLQ zw~iaLd0%Rkf;mI9P*ydV00-)x=qxhKB9D5CV^z{LjN|$Im(f#txybgdS-0`c`D%s- zCd$^j&Uw?H)Y!z-|FiS_&k*Ts@g;xwLi6{RR7$$*Vtg`_#x`*;snTVoOL2tGEy9f~ z83)w`9Xn#MRd8oVpLM)IM>U)dn-;vB87a+rG&Q(0HMf>jlxD<)J-{W8IJPYnLkx9) zjr)Eywsp~jejEQ;&p4nBJa?|bedEUQ(Zdqaa?0Q%z%Mas5uefZyisu60?s^`Z(K8z z(SSA=IdTZ*l%L1Rv!rOAR9%IHU}$QjP?%d-^;b_Z65QdalXj9>*>$cQtvsDdAiA*! zC-nCANPitLXi3IkVXfmP#ZCm7@yj5#BZF1#Y_!N`oQWZso98~c^W{Z1Ll5b2goAbF zFIcwM<0dJL@4PuC;#iH!ty#$z`$?Y~E4UTHAoHf2 zN)#971B3Nd5G-_8(*Beej7kNJr@Vd#43L-|&f`qpfZq{*5%P9beoLxKZBHsDe#XEu2o=+*}^Y zcU(3D9gZfSQHKKPMi6m~wEywr#}{iZ_$1$avEQgA=2@a#Bs*#i^q6Q6s(CSpU;u(C zfrK*8a!eXt%78AVWT8nIL`xWk5DYhxBY=Q%b!|=BA1S>cuUA(M+Y(SEQ;Fdn`Npag zbi8+f+G-c46w&E9R|>=AGwZ((dNPNfpAVd*qhm)2fo92Jc{UxUi;ses1cF1*gfXEc z2IrEKSqYP^;@`i2M+BSnv&3_%P#h?8*7!;&uk)IwfTi>ByDJF9T0d~7kV+)PB&RoA((+>j6_J^bo! zINlIST2*L1lmP;`sg$IsHXJrBC@4r*aH-*HGpnGWnsMjH)_s)MV{-MIrAtml(b!b- zL>jloNgxW{8Sb6@gCsfOOug}U4+e)jAnxl*igrl_Y0zDuCpt#HdBSG$q%Euy@JZv1 z;l}vR^tW%`wE8p4ySbT=vui- zJtHF{nNgM7TaJFq>uq$PPAe!YyJBU3mSV^3mH!z#Uc&(N5{GI=2dIjZ{4Pitjn?>; zY;UdeIj=3A-`68^<_x!4FCuTl!kOAz4LdGqI#{MQ3@7=MpW)89sVAu+dKQZxCHA^TmsGKvIc{YQbW1__Spj z!=xO(zoJ=$^+F75ZFM4={EXxF=JH8e+7wXas@eKb+f`=Tg?!Q58BWZjOCj=1pdEC_jZ9c&Wq=${TOJy*o_{ z2QGqh5}B{LfEaf*@CYD8V*|rr zRd4d^<)^oDJJ$Yd^)1THTOs*?zqMmB_HC;Pm%|B_#YCI z|9>yfcnTN6VyI*9O!eexaV34Gnj7V6)fc5?w`x#H6EMz>LPc#jCaz6aF zZVP($E51Po8Pd-s6LtL-HV`L?(piioh^%Z~ zy6&C_=4xtcOWeejXLY+l9TXgYL2Phmw8{7DxUzA%c%9g6$BnX#$%ooq2n(`yO|-aW zB{lrHsmi|TY6ITh-K`(KvMYswI>i50DgG^3d|&@7R_ozx)2phDNZBzPp)~kQ1y5Wk zlJKj6kIxF1XSmvPX&M1QeLMf8@HgT)`-Vt@laM?(v7h#jkz8!h+pD&_Q>3{DyRS+3 zmCPs!baqS`E%#v+!$ozBO%=mBFD6pcWgetFBfeUXv{4+|dGgax!`P{sSro?^cUbO_ zuBnf2>+7;H2h0w8dyg&sAo^|%L2<$TId0{kGIazx?Y|$Y8m_>T^*{%(C-!0DP z%d{%K$R|Fd5NJuRlhj4v$^0P01;hc&b%~qklSr6aclVLRw_t$vK=JM&ZK{BsmfvlVi^M9u@Df68XFUUxQz#2k`xZURx{ zO~NYwKaa;>C*`k~;g|kj4vpH`b>Cw%3k_!9O1Lub0nz{&InTC9oq;p?K=pMH0w*=~Bz@ z(B9S1T3uZYLie4YPZZDogpvzHO*cf9zh8PUag(s}0A2e#Qm%x0@AJG5}d`rHKgM#LlY5?b3kzx@-h(G~gf94-!*CyS!KmB#F4e@=~OU2)-NfGayf2;wDD0VNMsYX?f<)Y{zU;>yNZtxDfDQZ(oZG>{L8Ma!z7Od=`jD8| zTY7hA3u0~+jobh~SBdsv5FvZFy1q`+6MPox(OX+vs;cztdC+oWTp!G3)Sf8&jK1A} zbY>b?>hKt!K}*c1r;y(hz;V)qMF$2ZK)cne zmoGJx{F0yA&i1E(FJ?b~{uWd}vxKUD_YK7)5f4EO$6=lA9^oZ_h;B%;57K(spqHEA znNFtH9F-0!NTBS8WQG5iew@2_GLlc*xpouZWrLWuohxqvTY^_v_Avv!c8`Uf`l$J#t7A+>b!CkUmzTEolBvu)NF>HwXS#T?vov}l!-Yb`b!V1tg{gKFYR_}S&QPe^&l*YH-@&YI z!S9%VgSY=Zvp;_Iud%*+kYCq4O&2ldCrf@tbgAOug691_&payQuflyRHUb zQaLn6@zL)5-RLt9>10a3SDC1YCU@bm0d$r-rocZpXZ}4g5 zxto|6yz5~ux;ru4q_w$1oKf&|zTIW~^uLGxJm@znJ}my5$K3xX%X;xt`E0oH>bL3R zn17=5cRu$M-}Q^W1EGY-t?W&SnTEBdX~nW$+-9`l)NW~bvrZ^ewl)`u~cV+3e*709si2@HLbR}*lDzL(o~*rdjn@WN?Jzx zw}EB-IQH!0ZB}C{b244^_4V-IhV$bB&HfxbrMX3I5iIJ|#FrlhcrL-f#{7-AIgKo9 zaSyj8vPE)*C!rGl`X0m}@79-v(WGfvvhLf}0@P$!-8-uu9!ys^jgM7fYwpl+clJfPlh34MX0G3E796I|5=y`8>6>|eXnPGsm5 z6AeFxR)e}Gs-0kxueJ*v9?o}tng>9*(C@H*TI#xv-A@^_&tWRIU>Sgx+gC-}F2k9Nb?3Dro1K;m%pn?}RcduH^WeB#7yWS)#8zZ7a zv3sz7?*HS6NEjLaxT1e#ey8`({Vn=G|?lFxFybq-B@buC~V;VtSj2Q5;wTHOc8Fbuvpc0bJapw9RFzvs9T85H0^JYa4Q{|x`{ z10F(}qmqV3czRm>h*%(5z0L?KLZ7oOb|9Y>q}C;4!Rj5QcIFZi5(tmp3*FxTk+)qy z*s09TbOGx4`O^Zl26XqhiN-Qi#ObS<{pb}nw?=X`gX3M`>)W8k-5OfwIr#XnY(Lb~ zRg&KQc!RLoKwPWLl`lsT`{iY$uB(0c@S$XD4R7<6Lg2mTdE#ec;4>j~B_nugJXD_@ z&|l$@OKFtLnjy64N&Abg4DVxRW@dx`cU}3)nCH6ff@%|4?vIBQuD6O-&@y4Rc!J?1 z6%{Kq{f21ThCX`yIRA0=5u|w8~hypjI4B?%cg^-{aDajAD=E@95dC; ziX{YwoO1ci`ja91)wIE)k2E?^K!ZCAXll&{N~TiNGTMC4i6g!06lbm}K<9k9w2Ta& z4NjSYUPu|Cu`kMoCBT&kTGXE3(3DtKkltZPqn4by5iQ zBU%5XpAI)h5_4Q*ve*9=zNg5QrcJ021=Pk_M^b&Hu^pIwxGW_Gu2g_#14*DQ z7Q+JE;1&W5Os6f`Tr@IEH~I)%0Ok^}V5IMN{b#ao#NPr3o{#P|)*U#@?v6Zf_TxvPDp~)Tj6><&5Sba!6q*zLc zcUz~DqBEZSs51Qi$XWDE)^Kkjl~4aFgqkM|+LBSY>( zfzk(-ltyFtXM?(Y)}wD^P;SJ(<;Z9OP9x8JNEGswFScLus@}s`k>cHHv!a`@9o)%8 z?|tJWqcOdbl#-%i;QRLm)*@!jeiN@IGR$P1&Dh!5O+xgWBGmAgL0+7%35}$khW9j2 zYo8@<&Uxpm#^C5436hMFxj#_KjXs2HH+qm{N&Qbi^at`DgIk1l(cCE&jz2i_)gMqb zqB_(BciF__ehOW;BQCWZTK{Y~j97(vAiW8W~n|C>lS{3jvbp8}ZlQ-B&L^>29Y zqR;I6iiZ5)!ArXw@~0{(&flYU&%*rX9Y0-i0VBFxxVEIYb+m_sKnB4kL1 zL;pD{A}y9`e93nccu8wu{IK0y9%36ep=F`~kHkEr1WfwRKM2=ZIOvcD{TFnc!!>>v zkee3J_wOfz=_B#*g5*12CbY8hWPX>HlC<>Oz5DiIp%saeHppH2P$tRqy93y z55N3_QUneu%6Um`a}A#f4ew~6VhIys+iET@N7Cn8My7jmrC|O_CMF*d*8+84gjgjb zBfC<{yJM=KOgqB4si~DM<0 zeH6SMhxiN-CZs{tIxQ|H=rQe0LbxgmaO5@ z@-4o9!Bh|C>iZy7RFT!u0HdyqNEnkfa*G8x45Z}8GzFH{3Dy1tB$QHnN(c;C2OvQm zyyxUA8m*Rda-UooA_dWZtDO8(aBwjHwXdy4Ky{>|!$celOhtNtd;srizXnE#F9*kNYy($B zAw%?mbom0OEXF%E?vh{cfoZa4x{3h?=T}i-d?Pu%;2cCr7y95sgFae1%8LV#B z3Y>|lFx+)?xxj+|?AaU0Z8^{O9biJS$VVxIRAHtYQfE9A_)UJgsE0w}(hz_Q&=tOt zUUbZijEwo_Lt1dvlCqjwC~_GR6i0J+m z0FFWdpLQAt)U9naq<~}UDsl#!f3e3Z2)SGi@Qb3n@CME%${(0-0^DGP zgYa)Yc@DXojHi6jAMm9?pY3GJIjF!0JnaiU&&2d^!IqggRMg`Echaa54^B}fvUcqH zVS|7(9N^wIGzP<-hD}9)EnvZH()bW9hm=7FJQs4c*`7Unkp618wk{d`aHp^>Bo>R^ zkA0@#94mlO8K5k3lr^neCH&dt69{kt(^S=Ib7ko5L~eG1rqn(&F|Z`$v)~LM!ldb{ zS#T0?_go@E0yj6y14<9OqWgCLku#GmC#*nbfe2zCLMg(5ffo1#xWuhJwvUM5ZKn``~6c9f|61rDepV=+nJl`a>;0es;^+27H39}SA^526`&CYQrdLxoke-P!n2Y=|-uYa)U-}I}CWPj~o;EB5HzUHMaxUqC&Z+_zq z_o^CUwYHDH2GE1G$Ma`~=L&Y2$SnOkXG>X_lbW1im4oarRW@yUTU4Ej8n0izYW=`2 zt>Aj)A=8z#bvu(kEWkaW~P?I9-ZY3w_0&uYAWJb6|zv&|RN!xi`-^y~TTv z4TCWoyHZp3h~$uma4OkjWG zbe8XReg64%UFXWFn3?zc^?tqX`*}a_$Gz98P)DW|-oaoQf zeY~;G^Zj20cm*B%D$2QU1WTFV4_TyCl&*d{;`@IB%HNv2{)8JUSs#+erPcm7IAU||b_k)MsaembHc7oxrE{v)p53~2-YC15uv}?ZD_a{s=>cY}a!KJG| z)+FidYf$*($B%y=@;i;8SJs*yN>`F1zc^L4W-Pj2U%j~OtW07*BUPB$JyFT88}HMq z%|rj|IaP1w%ewe*9v?L)o21jxo$R$PbyqErO2Loj^1YIIE2V21c2$){_)<;qTI%F) z{|Ip(uzNUc=O^_TB=o-&iJEM3J@q6XsQUGoseQ({?XZ@#3ds0JnZvMLl6U*?(U{3( zA!8LTRgwG90hEThQS4qdG3{97PQ*XJuSc_!sL{S|mSJLkZe`u;zw#tHRWng*hJkNQ zCvUhqVk-U2{U_AEXeXAiZ!I+Z?aXn9H}vb(@BF8J@SiQ?L#U{E?Kb)C&Ygw>xzq9I z^Nl;CU+3LeDG|kNRrB(^@;nJ^IP%M~W$@o#jgC8tCu5z9n$jkwry|b@m8OpG>Q2`C zPgXU59CmKYZ|m#ge`iT`FF^=FiIZIjT@U0T;QnWKdpl8ZMGWt4sLSwo0>#lxod0!t z(7j3fH&uH4pXbx490H;B9*(PqT1#x{+90_G@AYUod2iR@gA;1*ERMH-un4J zq&2i$K;Exo=#o=WV7!_f5EHn4mB}m9X6BS4SbX z`15yZa_GwOV@=!RjQ&Wi9%HxEpA!o~lnKcmQ80Gw+B-8~m6($+1Pa~CbSDwqf^jBU z`io@~hrzOefv6Ijx{uRG9m!ezevI#fY{9k}%k2f;MAy%CI}i;3e5(E6sbuG%B?k#2 zPDaFLsw=a*wCAsy)aR!jjyqxkICjRo-2UtL01l@=vIu2Y+@K%~l9Uo)(mp&tf+QnI zLg(jqt0%{)U*8H?J7YQ!Kp^4CDv1bOKV2#J#h-#GVl2FP@ivqty|%tLB7WXNWj9Ps zO?K0cx{d$wavb#u%^-<^pA{)HV*4ObgRk*mF`-O`t0A~g#A1Ocj*!;cSB^mW6;Zel zKPD1@LU9Bszy|iA!7X zjSqYd6gIc~E2}TOQG_{ol!GG|5tJZZLMpqFg2sF(h-7+##YZ!Nz=pg6m{$r%h>&{O zcMc#*3Q%QHJug^<5v>n}Me5KX2t%BQD=N~00I?U`5!REMzt*n@Q*VlW(8uJw8p=<3 z2IA18pmBOHmN%9FJr)grj*mRApNs%3Axq95Sy29;!sO@~V)4viaaM)f0v?v5FUZ8m z`^~LdR?0!wnt(B&1(*?UP3vu|b3WefZUGhwurx@Z>1o`!@f5t-Kop@FEmf_&++HiK z1w&7#+dPD*h-i%%hL4=R)deN_2xvkFLg8GjV@((rm4v2g012Yz#}|5txQ!t?mjr2n z!CUs%ZsKU>b&?INNWWYK-5@=^fs!-{62THG!oh{+V?;VcB-dDR$C64aa#w&`QVoeI zLXb`H{K^7O1pYH;>SqGJ9EIhLr0$5yji}7zneM?VPej!P=B*qUdCK~Sf;r_(fNaHj z@$QqrprDA@*etk1w)3Cz!MXDqP^{U25>>)N;F}KFKB?c(5hQ_Ei&ZPC`Z+qjAe6F7 zxn+aF^;Jv^9-QFUbd6mjwl_Nc=OAGu0IC(zFU!f^UwbRux?KavkBPzLb|-F=c@Pm1 z)@TI5HfQCB=DKzc`a2*<&j|}_Lh^ukjuEGKRR02>RlIAYKqv_7&WJY#_?y;98bu}d zGMzXv-*p*E3E=%H7A!L=&Y=u)!vDhRLBvY%^&>q2@GQ)g3_0{NqkT;2F~8o8Nc`+W zJ1~nQ$(xo|Dw1Tv;&$5v$3kjfpB7l7bl`C;%1JUvL)ZEtZYu=mnUIhRa8_#wxmTZ6 zEZPBR5_zA1Gd_hp+gK4VvNa;^*9R~<_UUPpX1B?`YKX9E-+vEOXT&`X%CRbZJTCwr zbEC`}UW9Q030Mb7IX^O^v%$s&z9?%X-KG9o6)_FuDg3td4|$Ab+U%a>RP#STMAu8AP^lRJR+hG=IX z#&FG6t#pKQ{Y-gM5&20(L<@yNb$B4ia`<&6Xv&Jr4qY3Xz`}4F0+w+WBs%?<*ZKwu z2yoZbN?NI#>xiv?uIjW(3FOk1BuKezf=^aZCi1p@z`x(UNC8K`ZFLB;g`;T{-JH#v zFOHjvH^fOei&%A>Xyp9srRW|QM9cnT1vWTXZw2|yXS)!4PmM8izoFD2Kd)vU)!e;w zpD+2(ZEsFJv{H7jbubZ7`TnE0=cQ{c+TXp_6I`3D7Vle>*<>@1?a0|l9)IEMb@IR4{|dAoT%C(Q z1uUAoTI?nPe=E3kxVb~-0OktLuFv{5{Oc*mWcat#_@5B{H>Q6Ry#Evk{}bZ;37G57Q9GEM!GBsl)Kh#W?-P{UkNvzp6I9Doj_!h#a?7ak;+a!*!Ft?sV5Knn z`0F^IObv8=l zMRLeA?x0x|d%L#X9Zc}zZ(p#RNdYJ18IuGPdgK${bX*bR=Kk0uDXsJ~KB?OQ8 zS?#ZkU#0Mr$fgZgZR;BkxrkPtKrOC}V#5-0-zQ&NK&fb&nm&Xg1vkpOr>9L|UR*+A z0LT<2LnZE=30{szj-JQ{Ee@q|?OJUMDgWX@#t`)nj%Z3e?-?26i~0GgoWx!&Ml*Y& z@$r@(_1k(RkW&%grj(R2%j|56q@NKyw+v%%cR-_4;P!&Da|CQldjdop zZz}bFKW|~dUOM!3hLicm)2Ah7m`W87kFwK}uJ9@4o+>;6NvuR4{efAU@@H;^WkrYV zbllmHafSpc@6fRZo%4Hk))_B4`CE z5Y#QqOXTaW`>(tSY~hnnMl1Ufs#aGDaHeb z^X-FX`kJ{5kHnmY+p?!N#}qVGzUFPl2Ng9xpf?1rilGe-=Qgx?mg^iuSKSx$1A~LL z$DTZ^kDSag1o<;F>KFjVj!`YP-~CWk5)ve^?cKgQ&xgEWc`rQqYf3~E{R-bMEvqL5 z1ZXQYkk$>HncHG1+nek^!zuh?@wa5+kz|oGtACET$v&P*qIAiN=4{=MFc~RsbE7Ky zrMF(wzq}8MH+2iIv|r(hlG_{%wa-<4i{t6zo)ih^b53~wXlGiww%>iVk?Nr&p@s4B z-w%fDFZI2RnD*}7`|}UVs_>h;(>PhS>iM}o^eAQON znVwxsB98Oo95{*G!uKY-y5u5eIM**!D|@4sGynRxE22h zo;h-yi^tLRN02vAmhax3I12dMnb*fNwhp!TYWX&+u}Xh%@HV|VWq^TDtL5@*lZ`#J z(j#asTuhoq8hpCDcg+wbOxM&c5~x`YSy>JwgDR%_)r}0V|<|cpYR} zQRlC>RA6wyDUK~#(?HUB8S|QBBl5*h^RY*J6=GI2RvHsmu;gz?c~b^s0#y%N#E;06 z)h3E&wl6*`lpoOC%z)YgSlN;T4qaztA*ofc8w;|c;$#B+2sIK0E0){5o7LgzqLPe-Qp+Q5Jy#M71 zaQ(9%KHfX~^hOW?j02KO&(fHz_w-l{^^!kO{HRb?Epl#rDTzJ;K>;!c!gbKb zwN{W*@-?AqpUnd+{}vqw{Tsmx3sfKW?Z>YfbQ!>NiO&a%$Ad+Kz=)vVHx#T1x`~>#A9N*FTYcbBOh@}h_zjoly(3-+O}-cmp^{d5Ke9(g zh66146t+*M2UG<}t*JkLn0?#SE~4RC!QtHn?X#ySjlN|Tw5CG2fVfo_u5S5+I(F}- zBVbRW%ymi1#4pRcRjo~|5>rv|9+MOJgb18 z*MZ=cwDkw)@#9F62aIlc&Z_l<+#eI15ph-8Bw|k1{p3e{&!3}|W{7IfAMacFICAo0 zeuULh^_?vbR0MiHJiK%+JVaK5oyH-6``S4w>#-yQKc+s3cpcVnq<~>M)FVDNhcw7s zUhci^)#UAA#K^+lpcG*?N_(jpC1h>KSY7Q??zKLvLMvklA#pN(&h9+iv;)Ium0{e7uR_7*zx|%p^NhUo<@@LI@r?_Lwzi%mK;s)9&wY!jeYL-$_CxO zx$D@_(FuiV#-)^#mil{B60ONmfrIl<%e6}IOqXwjc@ejvCamtWwM3&6nVWZhC2|?p zaNy8q4t!$7x>m+E+bW~?I*huI2STZTipi0o645RGQ);?n=efO_D$6Yx%Q46KMQzgU zG`%tpQYS`-3;d-+3uPa})7%TvQJD6twA$RtN>tMMxW=@HcA8}N6Su=VS#NMILNe&?-%OA0()w zq49_Cx#3|Tydu6{@xtV})WZZTgC8ZfX=0~N&5dNJrJ#=-3rg#t(xik-^jFzt*EnC= zSPAgx;IM`|NkjgZ@p*qt%$M~3bt?$(*+zt4Is)t{heJ0V%cE8{YuBHk;VLicLaB1fiCn`$dOqT&Z3`w@Fh$CvgVSq) zI)`SqzY}{U=Y-yh6^0zT?i9d*C&Wdi2Ba(9aA0lOTbz^KeO1M?##Sop@^DZkIV8j9 zu&-;uwy1dg_n&*t2V<|y3QB*0t2_HnN$a&}f3eCJ$J!Tm4$LKg85$Hc3~W)!-kp%y zUC2-8k$R{0^O)*9p_zAL74k$L)wp}s@i?fDofDKzSZ8DG=Dwui47?d%%398P8J?C1 zMPu`Py`#*WW0^mXX1I_Hm~~Wy|9=07b;qlyHde-AJqFaT@VCb-gW?oy^r>m`|1>gXH zwAz)pwNW+Z@n^3pDH;CjnHf^4Tvn2)x%03-@A)43VvQ5Q>h zoMO#Al@!cL{C!WYPVSM|Fw5HN1E$&689n(+7daVwawGC|5;#WYe=VC&u$`^aVid8K z34ZYhl}L%-7Y>pR=Mepc`};m|^hjQ(zbTvCB&K5c$19Wj90@3o9G#r~RYPX&Stc<7-G7|D7?@DxhJD->qJbAJGsHgBUoNc%gXRFXVs-R8Xtldzdw~ z*+1s}W^h#ORt7I+y>>f|bA18xO*9Nz9iS7A4V`+G6)GWK`$bv$a{W@ z&G&bvVHP7!SLkf>XeU^o=OV9@g0CEy`#>UI)`2 z9u&`4V{G)e@$!&vKtF-CVQtOpVL0V0~aW)I!`jS;V zJS0DTDWGb~c&+u(2OZ-Fhs~^LUG?sL`$$M|lw#HCg+2m^wMqlCm*ms{>(DNP4N}cOQyCck;a z_0iKAL>doN{v9nlREGGttcQBC9=~7Fy0}@?tJ(|p0O)VOBw$8Zjm=ft~iKCyRe(d{l`VhYCyHhJ4 zc{nM~U|oW>K_@BT9-q}}sQil<-pW;7r>@&qnYTNd1Jx_>OBcdzCH4p9nsW@ zjzsxR_LkcY`t2+MFQ4UW+maIBzrO~)lt2#Kq(jt0m%8!P#JPwibXI!dEV9E6$2*c~_I^xCf-Gc5nFR%EiCKZ`5Q;~jGUlUyeiT)Q-PjH+!w z%b#f6SDJI^(Tb`PeY&!?oU^ZcZM}S#TxQ3RY``zkL$pL(>_rZv%o9G9g~S8=LdBJS zHX*2r5}FH-HbHu6(HHHQLD;feS^3_(%W5R+>hHT^eL10^@qw|rrx=lCx{gluJ8Ab7 zob65QEGus|J^x^kg{fiZvn>0KM{i1cMAtPS!`ZN-Xe=zl;(TA%h4f0#8B5~VhaX{R8@Cp*BMpfi2T>)(!s{Q z2;Yo3+T9NEe7-uVk?s21@AW+LK#n-#fQm4k-^I^YDoWVbiE2iRER;O;?CC8`Ej+8} zyH~t=Z(Wd&Yrz#^4UFLC)#LOc9c?>^ws+SLl#ptc3Gl=DaE0q_sq116GzO_6vj+=K zqs>^am%Hga!cC2zS$BM{Rm2Jga^^Kx=Upg2T_+_olohnMB%k&;>cTM-3-VM?Hct#^ zgC%>UtXIv~wQq{E1w(vM*R1coIMY*W5@jj)7d_=6iv9tDDWDamAaR^G`h+(@t88{|5KPmklwknj&4m-Zv2Jy%-C^)01^O2e_ z6ji_ZrFP8Hkh(2AKvrSexkBsT;)ASGHvamYF zi}xNeoogTjM0mdaTEUXx6cGM+2i=K@stY<;TbU5Q9!6sNsnp3AtB1fC z%#DKzDS?MKyWba`Cp+17{5y3BGhqd^;W1xdBPb2eTec_g%@j{B6;@B6FF8nDPr zNB4%?W@#D|o3{J&QCw_lvTZHhp^#UP{A158zjKZ#GzssB`g(rD*Wb-@WP&>{OfX@# z!fOX3*)FdasbC(O=($p2zmFzqRkU}vgd5Rb>IaEjc$z#SuHVus(7oB2Zqrnzc%z{d z=eqIe!O2?5rIn`bXHmBiIrxE>b|SlBajxAmEAF!{uc_YD_F#W(j>fmFqx~OOU7>=s zZdP0&7U~)q7GA+CRC*mQ>unQ<=lwN_xvBXX$PX@5e#}<*e|6rFBJV=#>ZGVfZBoQI zX}~|zDEVZ%+LTwpEG~kFu5wa4@#;O+9A5i5aAWGTDYiu69zOZOC#F5^YAD^dNH{VVTP2M= zWiA&jFT)=(s)Y5FdcMmT288cV(qZ9{I0h;`h$fv`izKpP8KLdDb-TtO_FQ z-UsPl{`%=ADYlVz@A7&+->>a_!EhkwkX6>TPmOXBx#emVnD$U)J0Td~Ga& z5hH!$H46?;h~qWBJd(x@iud|t&lNW{rI!M?a#WBif1~11jD4l_J+0}z^`F`vZy1O; z#!lQo>W5e4b6*>KLpxOES*?z^V3p8*U-R0ff}zwS3Tf5N9eKag(Ru8K2M}e(X2Qr@ zWOn6oj^2@Gdat(bMMcjeHTvU z$#r*gg%A7~I!D+q3HI<#=moZbY|W-8UZSS)T66IT-DiEV69bv@o&l`Vf&x|{JHRkh z)i1iPEhrbdbz{PjtxqRr1zU8#m@FT-g4soPoNwU+N6x1ehsTW@8VLV1uJ0@94BM*8*HRGS>8uZYt>O}(WKFa(D@F*LO;>TnZ|QqcPZ0@w=`OS- z9Puum{xG^Xgx;Fz%E=&BpZ33eV$v~{L0`T7aYGayb;LzO|gV3(8UBhc`X%jTt)D`~qfG07!XXxneNy$2yo~(3gW@O+v zGRm!ZhR4-e`)AwaX9MkbP}YC&7+Sw9Iofn%-Ex>9tL~@5wwqglp}RuwydDar0y?@2 z?<+UkbKiI_dbQ8Pyj&IXEO#Q*|EvB~jygXRel-2mge_XPIgF)<0A{B_7Vclgj0ZNu zehdSAuI)v8=71h;2@Zy;-)9Edpv_#US@+^6@J1l5 z8_+Z?N=lENB4IihBLJojbro8in=h>zahC%y4CkIGTXDZ%g;m1#)j}DVoX>fZj)7^` z=1VFnz3<@q6rR}lRVDUSSXl1N%y9-b*{Y6-iW_dX!KDfF-fApqYw=^ZHo==>UO2A;bqec9Ekb{E>d7HbKU{Qka?F*rh$x8Kr-OT8{V>+PF%uP9C9bzJH2M z#pG3*_`|5xN~){xvvLZR_Jv+ZOG&A~f!2x^S<0;>Nx$sm_o#SC-JAsvxfdncO4N0* zNHQdgRM^QhW;`x?LNyyHzypNt3b5O9{IbcOrBe4n&gxv4r`<5{6(}_sSj8)wIbeklms?J^OV;Lz>tzO1GRo&Exg{GQ;(osH? z(ydzs)Qg6(P>2T4sWq7N2jBWtk9scoX7qI(ClT{{VZ5NWiHXBcKQ6nZiy|7_=1=|o zZ}#R^X<&E~yVl`T&=xu+ErOebbx-_Rn^|Liqgae6O*S(#Z1-1nH zA-AQ$v=PYK5)*yXtrEOVn2mk%pED(=Q6^JD?4sn1tCbk#D48R2F~n7tuZNdtJ|;=r zxPLCmHorSU=-66r{n;dht{E;Fxtl+OxEX*R12u5kK;_EY%obYkg!rEi&SmarJy`ZQ z@a&&9fy%kj7+ai8dBl`P$!ouM?x?M3Oi)^RYNU=}S_Huk>r7#v6d zrN0<@@SyP#2g8}u?}$UP3dRpzoJN{fo{0=}Rcz$ib8&H5Z~gcV9rn6V6F~gWP#{$j zwruNyp}~33x)5>h_BEh+BMk_VLTEwMV6A_xjT~5uh(K`ULIH;+EH*aw86#(IAQL|x ztof9cPC3rD+k5HE2co|TV^i@2ybWQn*a-?NP9YDxb!+{0J>`(f?Mi7Rlltw`8AA-k z9Qy}Fvjj#HX`9gb$i;Oo(?kZ4{g2zIT``~@Onqg&DlNmA<2yqfGD%`wjySetc z(k`s>Ud_&LnXMWpz32LPsn7+8bfzJGXn@@C1Fa4#0-bgUGy<_kfP*|8yjn3OY(P-~ z6i<<9{a(c21h^T?1q$#Or&=N^JXg|@S_t$OV4&v`>D(~!oMHlprUHP}lpRCN+6V<5}Q!yt>>P%8q!4sWj{ zu!NCm$pB9hfj{t;E1(N)^cArPln$)SD##0Y$u0w|NAIsMMp=J6v+dunZquoO*FSS* zscJPH#w2s0=xetP{D-&S!l<4bDskVEa!tyy=?h@?Z|O!KSo(rrNni`kJy7JyF};&;=VS)?Q@7BBTcIj|fAfR+|+#anJl+ zjmw?%v50A`n$psWXVR0eVQR^;&x$mtvJM;YgQ)Cm$d|>KOP<)Xsz204F}pK$?l4q? z?F>*$Ek!0X z2`(@R6z0J~t5W!~HKnW)rupVP%&W~U$uWo-q**Mm7Gru1WT#q@x5mIGsTC!J79(LS z^Ln2jk$nZ1tk|BQafHVw6Fxz69yV_;c?qHXwMUC8uRfk@$?%b={D{&5rcHPd35A==Uc-5Tpb4d%RTJ%-AKE2>1;C>1ViX#0 z5tN~wZ=_GiF;>o7U&>pV?uagq*orZ#v5S6om?inaTu7EdM2i>ZGn-`|&Qv~N4%$=r zG}B+`^d(w9d_9;fW3D<{T=G9gLQ9%taO&e8AoU zSTnwqW*+3OM({7>P_DX)&fUHm^S2kkj52Lmt9H}B-fMN{6x;{o-xZFLB1WqxVWm(_ zyhRNgkFiJ{;m9=^`MGps#Ju=Mgob2kb}`6n+@QwEi1aqv`?A*1bougy{%c1WTF3o+ha>$#QT{A7e7?p30;Jzh!|`+e`^m_M!kZ7{eBk<%7&pF;CV8jOXGefn+|?2{paMrCgh3nPqT z?twxQ8GLcpK&JW0w?amz^XX*2hK`0J8A2=<-Ie|v{c0u6vuaovZCqpWR-K7YN2JDw z)N>;#DZs+fQAk%;H~s)4F?p#57CR5QH!#AWz#|{Lpxn*-9pyG=&bTfvUyE5&sFl+I zvej+mcxiVJnw*?W<%|*{vhCWdh@IW_4MqeIYL_rh&hDC23d4D?L!sWE1_$SYA3EA@ zSCv=XpLmvF+ozagP6EB?@zn6*;`^P1@tdN+7{J4rbi`+8Tx?O#A6mYB|X&b4T`m( zXK8u5Qz+xn9%L5x!f?4}z~N^zk7;a<{cEK#lm^-_VN^*kSa`^UEl&!mQ?BU#%CUjL zLHw(X4Am2Ixu_$ueXE)02n+|Yj%iAK#uY`8&>D@4mvki|`A>aM78rb63p4xBg0M<0Z>-*5jZ^V-vc79`gSm{Yzm z9Q}vldD{3sYQAsfZ2faC`@yo?j z76xTvr)1+{t_S;&TvnEDZl-$DHJFFg5342T_EJdSfU}jTR~vxtNYu^f8oF}^ecH%S zdSC~OWur5KV@p$U>ymEdbG=c|u(uQHM&DfxKdW{dYSh~mzBi0(ckJr%=?Bi(?WHJm zck1&aMKEK6J8gRcyN9}2lg=bZJ8x)c7$InE(D;&ru+OaF++^Jj-;1qVt%*{!XT?ih zSCN|sKGfOyXkk8m{3`^TJhVWWH|nR}F{*GubxMy69~CnR3GB(rtHPQJP@#vox~H-f zPCdNS)Qi>-W|%|t?2w->R_pDZt5Au-W@agk($Yua)X1T zn{j!e(Xd6$zJE%ezPU2p9-(Z?%;4*&S210aUFF_BFT?3ndy5u@_))F9Pa|l)H07xN zBQO{mz}lJvJK5M`UDPg%iHWhuMy4MrO*{`&t`wMIp9L!nv2=sIm8B#f@Y9H27r`Jy zSx+1GhFfEklbwlUcK+DkPguF&LP|V^pt*e zDoz3+C%P*~w!q_n{eDUx3zPSfLu+Z>VG}%)B#aNL%)V7ed26mIZGL`U#O}j>iOMqQ zn<=qNn@(6VFH!xrR7?68WzrK@z5>+SI|hEUJ>1pQj+611nbJjTv%{6g-y;O)Y@r%; zhznuX2`Swb=+1dOwuF`;T35SV&A#z~BhKSc-z10BK^8@B% zkDtDQ00z5LMs-)xRkUhu*@!xPx?-jfBDR`wd!oUXP!Y_9gGZ26?Cuk#SLaMi@&H|- zi^M%<4Zd%n*L$Y^`_0SJQz;><(;dcx%uebs+(+{MR+**WMR|=ilML{`#9VZaJj^M9 zKh}9XC8i@@CTsl}qwY+%+1x-DAWeV!_Whqk_H9J@V$= z!PxLZwbvo+^o`Y43d5lPw@_%uw4TTG=3!IoXAPC0&bjn5zjR?0Td+(7>U$NY5>J|e=ei}u9EAk@3Tkm$n}^w%n3)%ZjlA3w{}AZ}XYrEZt%tk)T?>orvrvc@tWg0D^*q6)rRPG7qyK z)?ViT0-m8bo4pCls_K;$H6f}+p| z8l0u;&$iXDH<`0Nt>RJ6cK)Oja zE>##l`Qk&n6FZ8)y;gf#x9w8FYL7KV0JVY?#jcRb3=KxMhBV2 z_8bzQnCLYUq$ep;w-f~mr)R493`@W&4{b@dgJoe2FF3f>goGBPeoJPrTt{wFEv-Za zONQ}IUNxFls2(#8j&z}&{Hw4SO zCGjzV9x@BMiAW@7x2GvbzJApCJe(&sIHCCK1|WM>8)-6WZ-uRZkCSo(8pOMoLPVb8 z;P5pbsa#iYk5NOmN%*ukv28coMuXxc6W=>sVlW4iE5b1XiZ9M?pa8io2W+AEybVVRk zdN^jyKi!0Vt*^*N6CoYIlc@`~IM?khPZT^d?TFGN>Nsiy(=tSdI-#c`3;qQ#a13A! zK^OoY<1NWdbe~7L(Frh*VK2Lk(}yBycnL^u6e4{2@X8s}#q*n-0Hci0DG@TM%7oS* zCG1x}iAF7N6c)it@+4;Bko?Yea{q+W70WX>j-?%5*91DHSfvJ!1i3s>yxXet_z@xu z;GDb27$QXpTaJ?f7Vm65dfAWrOqYOW$qO#C<=(!QSG*WEx$5m>yarmG?sM58vKv|? zF}prE=*hEIiM+Md)u6197sTl!m~fk{91cdTF^B~u(LY^HmZF&GLk^n_q69GuDYcA} ziWJ9BThLsUCdbF_R%+rqeNk}8Vb_y#ca|ejy&}z6PH0TL%P+joO*1IU)y8!nuT1E%wWYIeo z;J+NdIm#L)2c)mOsjjY8iV}=Br7a!x+jJs9>tC#i8uIpX(Y1oSQ=$>)t7}c! zCEZ?`zleMRvBy_*@RgF0LVck<4_|2+dCxk<9 zjquY2?8Q9PvNQj@R)hbvv60c_oUM^$;ub3y!$k@C_QR>q0o=@V1#UTa z?_Q$)v84)Q73Mx|>+g{^QQ1XVB{&+E;*vu`1|*v;b_OyFIk+8q1cOEbrW3-^1iU3F zqbAkA;l-@Ynoau(L7e_r6Q~jfB_0Oow%3CD_9-x(Q$IPfCV@UByhk%$Zhrn!ZJ;!N zfw%09#c-RG;JO_{pc$5}uxMRDgJIy5HG+$$4!#g%$$)Ez1lec0G8M@*i6-7Eh=UI< zo-53F1!x7F0beUTz9%b1p0}|r1h@!7AVCxr=##=vEQtTcQaldl3UV?*n;V#c5e$qVnFI`QeLOSvud&!PQpLpV`LD1kv7 zXVs$l6ciNT#acw(>tOdDp2-7E*`|)^YCQ&Ar^@rOH4Wz$$9}<|YHH zHH1ac{r6$iZ$^i)7WKZ_r6kgAVjG|jN(|6s0|1Ix5fGS-Cd^j#k9|-attbU92r?tmzns^fu4H? zdiFrhm|ewER=JwBlfa9Ip+@pw8A-;oP)H`v+PO*(Bled+7OL99@C^{Tr=xB;))goNit!A8^$z!jzQ%Y+kAkrYuE z5N?gjWj1Va&kH?JMG zoh`7yf3Y&bPvkdpgmVEPExP7P)QcXiEZv-*@%pb8Zb*oYU#i*);JoFkm*ZQ5q-RMq zwqz(LVBU$3*$yqeV-fDh(Lk`wBZ{F$d+2{PBl~ z{h$akaTGe)2r#yV>G<*2IcuEP+34t6SuVL`J`28Jd3dzKqHtCN&^2mG*c0;$+GUEr zgQlGt%_}R6gd`tprS@h>sw)4@o}9;upz8M8bAA~I|b3GY`)Hm?>$^a zs(}IX+9}VzCrqb6YoF^v43|wuwmy^y!h?f@=lXLnCG8+quMRmP1gO+B$eu8E$uDDPE`l)VfhkP~^t%O)2rZz(P~bawY)VXsvA6a#!TH#kC3@h`egIZuLPh%cM1yIaKPnWI_eX78Z-1 zNL0ek^)s}rEy?-4zYhIGvv3z|d+}*v_QRQoo=sXy29eI?th5naO)tkE_fQQMZOu^@ zLYw2>2tDtP;-7wHGWBi+Jhp7BneM|J=BH6s@9S6FdfN$^S9k5PGX4A~Yyg8rqGy+4 zE^N~Pv6cxwaX#9@ZSOLF0-hyme<$S&eY~`%21I6}4)d=MDX)&;D*RoM>u}(}0akIh zY(T%c3d@>6%7-<7Ap)g@*W`U2l%Nj?V)A`_&7^wtP}qnh)*)U!klRe@?} zy*Z-$^5$iTonRZjqMK*b9i#nK)HF+XUDwas8@US|3qog96QpsfovwZS)!wr+VJVg* z_t`=1JQ!T`fE6Zop9x?DG#r-DscV|?{?AZDw}>xaI*F(-3~R!`~6B%XFx zQu3mEIBymrW66qR{R;^c-SP*Flre+{g9u__u~jDrFrxT}7UcdEqoa#7gZQ-p0+(h< z*Pq3huJ#`uglwA^r=f{etC%lhO1B)V&rk&|rxrN-=KTc_zV<_6oAG_3ffusAgZnRx zw)tPMe1X?p)oeYchBTj8ZyHd}Xor?^`Da2$d5Ea#7R|yYHu;9JsOOj65dsT2P!^pT zBxEg<$vymeHGIwcgRlzuYo6t$He^fzgr|6)=pMX!$&0yt1OgG=9>pa?$P6qG;CD5ggb48~1w6Ojb1 ztFmGDt-PA$z!+(Qx;7PW-9D!sSizA7Xs%^R-w)=~8({{PowwJ&+^mFhmrj&w!|t;eVy89?wHaF2CJ+@xt45 zA75X>+i^|Xp^p9MHmle09P3%hF*~*;{pMfC#m9?!&404v4B*qoTYs74g}FstP}5z# zOJvq6brV2-oJ~BqA$wn}gD7O{;r(O|#&m1p?(TsP^J%`v9)(BOK8z7{%mp~LsI5A> zJw`-lga?D{4t#)wpFI&9y~h56=egA(xwiD@3+_aEsI_G|EEw>ibF;q!J#JQG zwe6_D9De*vaP2JGUlbkT>#H?DRLIiFeFYJ3Lon@viXjp-EW|v-tLJ9dRLgX)eLf|R z>Y#ey=oPZ1MS*o`2Lm!WFUx;Q$D>uZrt)^CCMqGC1!2IB|=Eo z?G*At@<;ta|6%~p zLMBGyj-y(TzRXOhW!Mb=VC>BvtF}_B-fp-8CE;cO#=`>aVlK3d<#6nJ zlO*Oan*Fj zxQ5JX=KCB0sZ@eGN`mvP*sI;ZbGYXEE8=X*nMx3)qh%D(o@pVr{XW-_L(aWpKkpc1 z=_+y3`beqf;m$o;ExnpZZm<+3AWm?x8^9NT^{>9NI!BBO4-bD2gVtait=FobK6)%6 z;2O5(wWy()+;JY-13FL^U|hW>d!o0Aq5mc|L1<-uFekR(aTuz6QZ2+%EQ~?WD$wGU z8LFv9zRkhGfwGa2cq=MQE#{IoA`5|Dhs^rg-ep3pD;-tsT^#mYS?!lvYHA%aYz$!! zcc1NHm9#5625&fKc^?Z-yRyV4`917X+IfCdZwh-@4FpBAS$pWU#m^}U3JUV+mc$`K zlm_l>K-;ZOlj_xxFWh*xqg<6f8;C-U>`{IOGM}xB zCShU*sKWz=l&Q7d=p_cuTwYG!^Kglv^}h)&Q8n7{Mo`rG_6wPZ@w#eiqPAQpR=LGp zP${U#PiG;!uGOOS21N%8SG)x@iHIyBQhUG_tS2xv1(G{_dq&18E!`j!unK?m?)5gz z(9>i+$$o;Fq-b252V&7|E@;RndKbV5k4x2EJ4?Tl?&LrjV}or!Gdfdsq{4tA|24|r zzz`leWN@^XRad2#ch^ab`#!}pLUN$zT|Natcj1Qc`PtQN>gtM`9DXQB$o73|v(D_U zMK$mb`0yW}lBhwKNVlFz-68F@yu3la{YA4hU_N`~z{w0Qf3t&hSKI~$kEhH+oINs&EQwUO0zkcrQH zA=kXwDp5@r%1KJnYp;tdccunSXHLnOR9Ao!a*~r1?YcD3J2>*r_a5kj6%1_Ls3}L{ z#XX_o_ho%uesXh$-+{o{dtHn1aiC5G7i$w=LP{iY;UhCaApUv$w{Q8y#e@k$mSH6u zozsqvjF6X=dka9|D$$brWzG5=7G-*p?f7x)X1`y5Qgv;Fh~^-TRF;D5ngPrd+r{CZ6;Y7eFkaFyPY{6+0}w?N zMlm4{6F?+mQ{jMbkCV_uXqrfK35?H#5hlwVhxaTtLMu_RP48Kx7>9)Hz%>U@Yxao1 zU!U`&ZaJp;5tNIw^z!e{%IXxDqX3FxZo>mK-EN4AM1eO5VhNK}wF>PMO7DYhTd0d1hjjrp%7_rPd8_o1DU zk<01~j zW#{*m;lM$H2dAE;sD-986oi2%g|K9xkT~a0+f0Njp8+%#$)U|_{m5%0{v{w;!&Qo0 zl!Sv0F*@0#+5vb;`^}Xd^y^3edfg_OMM<14^soUgCkmhZU?6!UG04VBLVo)?O3@1Yj$f0V2$A0`kiq-FSOg=T&cVo77@kYiTC4+RXI}MM%&0%j7v$e_r%qi( z6ey;fpXot?lB-XfY!+w|`-Bh>rPIYY;|_a890-oJpC|9;sd zqc*o^(hYfP+@QbX=PzM5P;v9!I3L{d^zbnEjs@Gr^cA(g{_cUdl`dY37qh;jNefDO z#g)IlXPB97K>Z(B&-qg*)W0Xp@2J!N5WJQD9q;o0>iY6{sP_N=Ze3SJx$Z637Rpj8 zOU1}iDy4>yvF~J%rIB4GOH!$b#)OEjn5-Gb7Gq1-l;=;vn4h^t~y{r+|-g|N@Lw>EA!UZ3M1Q!?JOCvi~H$Cc#bV7Wo3ST>wgGw z0!kYcZ6DG==gziDvSr~@t4kyI-?QLeJrPpwyL`&TetMiSr}ib`dB2qi%ZC}dB9CLt zOA)V)0yQ!33;Vned<^gV!FW(sdb-oB78 z*C%iK*QFSQ^5}628Mx*;gB6|)7@Sw8uOQpxu1QN>XwG=8{*;e@G8*GRCN9(3 zm8epjcWo7~$C7yCmAzc*S#VGAB)|L-2Vu*(F*EGQGs=cIWbuwLCw{C6`^(jvzVI^? zXWw|#p^`Y0F!sScO78JRo*|-bJ;NV9V)!k1pNY>~-pSt_y>SHM2SFxE@^v593Pq!n zyhdDq96!(zh7m)FRgYY2&i%^ITBd{n^vBkXJ~72ZWQAx-Z?){LY5BxuH`VU@FOQj+ zPE|P2W!fFDO)_DP9&==w22DP1_7co}rzC>cqw3_=-rH+2aeyQ`WofvV{!ofNt8)nq z@j=+arv>Iqzx4iD6Q?(Yj$%Z27wYv`T@2sH8@cFVd{)o?lobBjAk$9zL)!MFBKMr@ z4DnOTOiH-xEP0@kY9kSPNX<$tc-7_6J}-%HwBKq!dqn1Ttv=Z7B%ezH!iRV0p_3Z) zlUh)#G&&Kp@bu2( zJA*>y@&!K^D3al5d3>|}QhR1AyL4o9f-*+;#=K3<*{au~=z|@nOlqqin$tT}H=MRZ z?{Ykq@t$a@fSXuspwPm7_kCTo?AeY39aWa2@*lw_n5 zV-DBg9?X-SA=5k3ag z^X-xB}N!NM&3tp+p zomt)y2Wjr>F+9Tb0A+xUr994<)l4zhH}7Vg=snUQoZ_``8BJ$Vn-wg!vN1EncTiF# zkKJPTq#VwTtA^X9t(E>*i&xXrZ|@YM$>g;putXY)p>FHV4$jX%RV-*IzZaMMS72qK zt3C0Hc}>ToCnsL*J5ENYw&1@85iOz^Ir_4dPb9uBQg7~SAz9~~nS0py?o=#Y*Ue{{ zNWEM!*?r_dmlaQ$($JskTFl}Xn#0!vZlxO=B)-#Gnt%zOv077;e!z>V#b+vWSt(D1tO`{f#QIMz&84g=Il2Neou`3yubO}XbIsxX z>B<-%iD?aUiD6#{w;oZlu;JA_SK&dg>Tbn~0^8VqUYd>z5>G99tw&moVbNQhPt2j0 z``l7<#QE8MS08CF4d5-h)&4zK!!DC`$V2;+{_VRO4@o7sGJmzdT8M*QE&=Tcm0_cY zpH#)Ze;pgLFD&T+&QYbSEQ-V#NQzJiR8JGk50ELSou}eejOpXEEi^+CoBW#vchVav zU(}(i`m!b-Z^AT7WNnvEdtOXp6QD(fQ;v%&?zX$`NA4~x^E0iKv@m*XG3Dm2dGTc^ z_hiB^k=*`LBiD&ew}gfEA2}itedtU>!*}_2UZr1*=uP0!%C*8E%MZMm?#v!pzzBym z@-8XJ95~95CAqyU?V5R`)pu55fzzad@4Dt6t#lJu>xU%O!vcbrg0zIIL$69I?$_Bv z$Rm{cPQ1WG$lxvKJbDM zjxkayC5;T`tMox9*~?|hxJP0o;lz?t-2jm_X8vh?|9M667dlaUC567_Mc(Tm=}mF; zxf%wfr#KA1oPuN%@|jLhR;psSo1%O|IL&%K+rjyyVnerz;nk`N-Adi{uieDp6Jidc zM_qUXVE9!@W`2D}yJ;|I>P+9ZO#2K(b5h#0N(@HNmW2Bpysk}YksfmnS$zF#Z%R4T+6N6A*;ST*Z|MjO3d-5qut{@OMDKm>tMg`Ew^7jCd3^Oq zpGX1feS({Zoqg3s69VfI?L8-}_HlO|i#mIK?lAe5nBbX)WBqk+_wNXJ6^jpCDrgbq ztY)He{OPX6oMvd^`K<~ArukZu!UNT`BYd@#InK~*xK$|_`HQVl|*G=WXFnJH-F_RryL@(79~be}f}v2v7_l%@_lPX4VirwBh1`lmT+L0|l-Mue_EL zLCWSJ8u9*sj5i6wN<2w5u!r~15ZcQ(t z2wkgoQxFbcdcr0QK>?!b3EKVqY5?NlI$JYU0e68o`lOwCZ}DAVF!r8V=mU2L@G6Qz zE(k=d*%Ai5ZS*vZimLHOQ)8N*47s;}S=hBA<+%~U@Av{v+EWNquDE+5r1 zwX%u_u}m-`P=+Axd2(_xVj}?zm1-bt0>q~U(T@Ncl8*&8%_+k6_VyrGdcHh}H%!3u zd*8z)aL0mLIWZG(<|r6|KoIW48bTOC9PmtRKKipJX(kdNs*`XWxPnBX5HJBhs6PCJ zxrF5>c25XeJU|#%5RGKiKnxa+)J(sA36ZFU_&HcW(ysLyat{zK4^B+Hg&6jXM669n z(Ct|@peD|N-B>cHP#}HUmgmOv3Lp{0&H9b`EO;m!fzY*UU5wsVmTu3$U=rVgMNU?3)D z-I$;rA>)!HfN1F;P!oV4kkE(+SY+(c;b`}35Bbae{QN39Iy#)a!FHS@6+b-lLf*R& zJUC<#(=TAdAWFkXaS>D(pp_QaJv2X-mm5&&FG`;m%e%25AKpWf6iGgUdo?fvLlD*8 z$^;ezfO-&n9w3^I0@@^igr}OoBUFjS3ie|b0k1g<;|`vEm;lh-5L+C0irQd4Juo#T zhDPj(Pl-6CxlZUyf-x^70-gx~vrz>^pd}e7@K1HPrGd#wQM3$zf~(xSpUlu`lF3bD zfVu-8+rqj7#BaaWlay`1bVQ&({Gp#;`1ZDeb@Ju=Kd%LG1ZQv9GMo;F2pFQnBXF*f zR4b@MFzCo^Vg*3((_}|`yA4Q0dw?eOr%)Jp?c;^<@m6^gUAeW5S=T}cg!*x@tcN7* zT{v%5s0dySe$b8}Ab>Z#Xq$xw%KN9%=Uv4|&!%9Ny^L_^DQ0s)#8eoet{_H#keaIr zKcN8Tv|50`sEoC?wkE)zBc$x3A>9~z50ZyaaEIhY!6qGG3n9dz2uxJKV59*YNDiZs z#2mmAF0@gt3{RA8^HnaC-9JDgOaz5rTOd-})V(fUUCCo1HI~PNUC9azaPiXJCb3a2?GtO_LFq@auE>>hQZkaobiSLN5Bt~piB?J9?=A2j}gJ# zyu2h>v}Ew}DS6H#uYr|;2qnP~4iTVDb{Qbbez1aR<-0p*7yv0p8xRj;uSE-_oB&D( zTR`7NWM&McH-c)RD#dwm)&dbThnO9xl7U(P$ihUZZttR7qRjpp8ONO3)B|2DVD#kV zJPg)kb-FF`0yZ@Y&*$A@0;VJg+erXoU;vkU@I>7-Lv)mr*Qoz zl)^tMOLp4;5k%)UqpmB@h-f}4egUWKGWhI6dd69}{N^9ULpC6HIm~HS`t|D-HZ3{R zwgmmt`~^1sK+?8EaDP7Hr-Og~N)6y?g`N~#p1HP*nK^%~pPNdp9{)}su0cu6JMTPESXBZN{?q*p zbBXvp@l{<7Gu6`jFlz033~3HsmTsmr8Ac~_{~_lS&NMeZY-{C_C}n5V#ScO zZIRM0n7gY!M9Qzmg~HVk{wBGIwA1V6d)3R~R0Cg)M_LPs%U&)5QRG4nYJP6%rW*fc+aj-uf_Iv1@4a<2aPK0gvUC{6}dVt zJ&4czfYlqmLu zaZ>}w+hMg9ot%~ z?XG5|aNJK{kV(qY|2@V|QT2c-H6SAW$6tM#bhEMNJYmttKA#&m(%V+wLm5 zb8}pt{3s6_Ag3C*fs4Tbzfwd?>zi+95!ucJtPV>y2GPf=2p-dQ#gOMNv{jtKm^LgkshY=ByrB`Xi$H!@@3e(Z>UezXf1RYFT{W9u;;fqmA3%rJRM(?WE+Y_=&89>Uv)Xg8ko` z@7KGRmeQoNYgY=L>N5X~v{?8q7SeX+Vx=tDER+#DAnFJfACgb>k943yB;Em$cmZb?OYF3)V!6y;DhBH`uMOze z2S0!7H_9_B46eo2Hek1rxayKk3KD`bXERAv@yk_}T&S7SQL&nvS>>j$P1_d6`LVL_`zSB{lBiho)zK@WrIUp!@O#S%WKk(z z&v7H4P>;IJUG9CV^p|wqXJ*!#X%~#LLKPqDzh|l<9Xcq1N~EGi9z!pX(VSpIkxFV}Ug5a+dkOo_`&efg|PZ`R6mm9yTSUz&J= zxst$oOMW!+VPvATm43l<%o!f9rs@^s+}eq5|Y-AORKSuy+m<8 zw_MTp@JMddH#ZCmplQk$N)#AstPJ-kNOdVy&99-xqUKnBJl*1(OrNB%!-thcn(CXp z_js?qvLb!hEA0NoH2enBvmuet?j8|~p$MI1a+c(kz*kpNUy-!Ne*4hW%-R1En_oFP zF`+v$znml9<@Sv}f+x@4<*0bzr(~ORWol~1D`;%^y$^=aj0+uIJG#2C)Y>AYT{soG zwifojznx!^Z`m`aaZalLrO3yxHCJ%S-Ub1~+x@EtXWi)1E+_gL}$J-(zgDtFV^)4cEDH>qgb?;qXGwNDO3WWE1dQ`2d)etoFZxW)_1Rm=76 z$BToHj{(Pbj3egy8gD=?ZGC32S5b>F)3z3?99TQ9aDlX>wB9sw?9?#lFC*uCMwf1j z2kc>z@@=T|OW%PbdH+&zJ^lNJ$B(Ih@m{PT|8c|6Q|~NCpYu#b>M`V1verL{s(<8| zxE$axsZ&=UTM4eClKKmnb$YkRb5YODm&ymW{lh=YCX~7!mFp7lypmPQgnwSOoc@;Y zb;6;wz3J&$&`{dFxz`cjwYljE-?IC;+9$M`pUK#=4MN}HtkBlhuEOOc$H!kLXc`%b zP`gsIvr%Xa9#0*2rA9e=#!~wHg7`!}bG#Vj*PZ9(mAe)T@dJs_{ zD&Rnc1M@4B(BJ<-vTTsuFLC1Dzkje{hH-n_>(__S7;yVg+i1V|$ydKokWKW}tDnHd z3OLCcMn+NDxUQq|4T%sS8iMvcm}mrl==%psx97mwFd;iz8|VT^wJqXkL7|)kCPUJj zH;O=e)v>oHgYo#+#kYlpXQ9vy#Br|Tatsi@J9wqj>%Ed=V$MSbuMXlT#Kals$`yUL z67us6MMOjjUuM@{1?mIb(Kl`&Vd}w)M&?$p4^So$gBQdS6d}7(-N-G`M)~gQAaJ9h zpl}bt1|VcX`W|sn_VzA<`i#b=CeoKL*P+~H1Y8TRsi_4a{w<)bdIQ*NfD&k&KYtfd zyy@&Tgw#E_%Sg!YgiJKU0Q%y2O@81WHy~^TLFLfz=^C;9j=3uvj`reyYCH5Q-Ibe1 zoRi>+NPRp=ko@Z&PIXmLjn~lA)BAANt*?XuYPSA03(?jNJ-afk0QqHM@a6$(8XR8_ zh)92~H!w7i20~^z4g##P+~Md3uml>sby~u6$ypMGs`VkQk#L&>8 zG2!J;5G5hLwrB$Mji#ZY+?Ul4v=k6p2MR`>2QBIB^7tu)^b5^LQq0&aeJv2jot&KH zDuat-+}v_(Y;Bv`+7hApKuH6=PhA4y1RN2;2}&O5y$=;`xbv;a{F0UyrH z{iC3tVHr3-)gy*ZN=m|bc;s2V@B@a$zc%#TdmmynG@2kb1q=$Uc7{fTlhPnoPbQHr z0oSZBno*BQh@{2HE4P@uw}pF#S9bDyp6C03*Y|!^`wurpY{bf!u&t9Kml| zhEueYeSLl9R(aTyi;BEI_Rx3Yw`^0SiUylMmX-oR1_LyZ=yR@c3L9%{^*eV~61rEo z#l6?=fbAzT??l6tAwbU^)(7MEq~zqL&dwANe$@tFSZzzoHv;_I3ZW_(DP;kJR0}Js zLGzY%+ z=###m0ARER$KTz1_GrOF0@vKBQJVQf^K{)zAcJdZYaay@B`6~8Pl(2YMbLgQ<=THt z`v%m7f+y)Ks5yHJZYv1)NbT%moChBNP_F;h={Z0GkAWXMf*8Rf3{(gywg&n}e^QSv zCv2)VN;Mu3vzpZ5RH=x~G%G`jpaYAVx<5*!k0U zsRh+=ZFph&toOVAxe-42=xy>8iUqc~0Dt_?cacCv55qR|(m}_?&9~#u?Ro(QsK}Zg zLHayZ2w<;}iBioNWq11;wGMBC%^AGn+kyYr)Z9|Ae|G_W6D`h-+Qc0~etM~G-4!!V zYU^#dWpa6Si)<&f1v-cPdYjtoEK`%h|F?jU&{OxQ?!a-&Rrb;Wihuu=!Va@;KIO;Bio5QA1YNv;KmVUw`Jb&3 aSl{vXbnWii8p~$nx9Cf%7gH~o`u-oRGdY?7 literal 0 HcmV?d00001 diff --git a/docs/ae/kafka-src-pkt.png b/docs/ae/kafka-src-pkt.png new file mode 100644 index 0000000000000000000000000000000000000000..22540bdd0f07ac4bbfcf3c5c9bf5541e3bae2f00 GIT binary patch literal 49479 zcmbTe1z6N;_b$xVEh-Y*KmkRz3P^WIiwa0LNFzv>bcZ0KD4?V?NHc`A)F3J#odeP+ z-4a8`w|)*gzVGaFo&Wi~uJ_%*%>3qw^{jQTd)@1Kp&%zgL_k4+hlfWbdH)W%`?c#SX~-k*4q_e51(;ueNo-?mO3HmvkTbM6|m3DmcIBTsw8OZxfw1trfE zA$kV)H}`22?{u|CW-gaeYZR)KzHUNO-}#DeF*bgSc^hr-(LNz$-p;XY zzr20g%UgJQrlKlu_h1Nr)W_Udk^rSH4Ff~|S^9mqV{=(pu92#|I2IEZAOEt)yCIlz z^-PYE3p^63RO1z{Ge0+QnPN#8yGYZx*yM+4@FKL3^@Q<<_=_r=jn#2A4Qkzj0 zN%OAM7YSZFW_{y59T#{d*lh3+}NBx))s6-n_XX z!9YisdCQ!>STc;Zv9C{EIYTa@R$sx+u5iR_*NX4ftxVUosXF0^dIYxbQ6dn9IVqda{)H9cEr9xXsa%6XHvT1_T7?dSX@gm%R@e zC+8mb`=7aVfr?6R;Al<$q!LSDe{f-Ky1lQ$aeiWAf?7gC0z2O)k3p&Tb*4yQx2F^5 zhn+`D);%ZQNN^c7T{!Xk@ApG5GHX)%)g}d8b^AIOr&VgHc<%X&Mg5Mm1JngAwynjo(YbF0hB3mP6|m6`Kfas}x+snYN zwSa&CC7<2Bjg5_gVzaOc=Vcbjo&C*G?2nhMQ~WlTmN}n3eOkS;*7S$%`b;MQ2}vI+ zAI%`CS>sg=(=|DLLs>~_ebA=HZ0Lh62St!(xm_B1Z&ez*JgS!=7vHry*^KR!plUSb znj0#&r{FSuZQP!an|wE5syy7KIg;58Y^5h(Pox=p>CxuW@D5Ca+pLS4Pe6c?pI_4k zy`3(+J#k+4E!(v|r5~h>YG3u)Oq;^#BX>Dr={>i;2l(#@OG#;K$Dt1o+{-H~n_)GZ zqgeZ32CUxRhg~d3=R}=uRp|%2VST3e;X$Ox4T<0uOoJtl#cWp^1)ILl!h+qiOx@}V z*p&=0E$Ps?-dxn@021j?9;<;=Vb2}Y-HpY__J%KCGBr!g%iH3kqoXa>!cg)2j#61! zS+KKVX}PkoJQWVJp>py3MrQ6C{U%0S6f}~#nk8MeekW-rnf$Qp>%~{rHa2>He*YX1 zA0NN`)^V<<6RcN9Uq@ZD!Xe8IJ7-jW(c9a*#AZ~tv$NB6LtaKkWwI%Pd3pK6I5Bpn zGo@oZ;%R0k9kW)cjIOTkTvysle&;178ylPbpPxzi-8U@FqE6!WpYcNT)Q#!(#Hfsn z3~g=goPoM ziep;=BvfjC_(XOk0sj88Fo0~e+?cqyIHu-6n#RUP$=TW2&{TXPim#)idhj+S*ilTj zDkCOeujV7WkdV+BL9deYH|6NKxO%b|!46u>yrWojxQtuK_TlK7_ZOyV7MmsuELRoX zH@uk9ogr@l(_O`4Rl&(lj*a~(i3~?A=fSTLQVKY!L@=mIRh3%o>*D;BrD6dSjgx~z zu_d8hT22m=k+HE<=u*WzOZDc6JMvjULBU5hHuP{)mBvMJ_Kd&wRL|WW;k*C;pc0otyFr;i-2{xGgO$Nl8l=nKw5z z6;=@w5Ol4KH{=G}41M_JwCrCAxo^yLrAe=@tSG{1=Cb{9&vT{jQeRHW2Qe`*Gwd9O z;Ph$50Ak9VJGIRVi;I{7Lx~gk_~7?dnHX`7pYC`d4Ow_<34*OsJi41e^+~ zH*elV3KL!C>Dk*}W3JV|ecR{9_&A&kW?tSh!S>5nuH+;o{mJ~`{l~7zgoM1tu$%KM z4OCIos=1n~SFc`;oF|CF>Bv?^ zF)bS!7^o!Q@wd>Gwq09WQwG~6J$v@eG*fHyEAbQvV@ZS7mANuYsf53_P~RfZO8tlR zaK$V*NToz!Pb6HKn3z;UoLK$&^ZmLR?9|56um#ow4YsAlv;Vc-BELGlUF1+ZziKrQ ziNstKSUAThjUwYn;m^;Ay2I&}iy=xt&^dA9goYC}I`Zew$8}%6n2vpX0?xLnsi|wN zUBo7<$f!Azg5M$2X>lM844<{#{^_%4+3?4?!Locf{S)KkDe%eW6>u^192`pfyPHvA zVPWu|1iN#kG_v1(WvxLWRKk_`+{{#2ad(q$ha6kDFT_?i@HayEe zYX8LjQQvQYwdTHz3{xw=&z;k!PSLHJaBy%0CZGHDvX53Q4Oxy-nv_}ZZx;d^A2F<& zfa5xPmF`MdW9srn)~;vRdB=4YB?%C$Jwj9YHDK2OkIbu zm;C!Jd)w#x%rH$6{7&nc94tCYX@Lg(z{qGWa7XH1tOBvi2T&`VTv=^)GmDjI0o zr@MILhLK6sF1=G0J^hZ?w%vZ=Lq7w9casYXIb&mgIL>+0z$4>P-ZC<;+=SMO6TCJw zgjQ#~S95Kfr(}1wQW|467t46PYI%Hdv-c5X(kqogX6QaXQnRF>5UO^tm6G5xXI5gW z!SS22t_*G12$g`E#DQ?m;rdJ#{{H2!t;0I(?46p`c4<2`d!gd5DAL4(mgq_-q+f!& zO_z(ccD3P&C3RkkJJ{NHmQCPN^rpO(G@am$zQ9=JwKm<_JtE}JWjyr3fba0Yb9cQr z_68=~(SnGaU*@;pev^`w#eDZW$-RkUEphD@uiow*gcA_ieosPTKKIRyiRonkZ|*2C z@TRX{JC=rPEb5ER2GCZqJ49q}O_H{->UvokMN{`kn`{^_A*`WLl1&7P) zNM?<9c_c#N7A^2_i&cp*u7_O`uJ{3T{7ct=nY;S8;uYTg(QtYS9*+;!Ha#=X+7M?7 z0MC85D?Vy%dZgOCBk74Mol9D9{8Ch-Wp5j$fYgV=!d%e@Z|8bZ8LgLjBb?}s zy?gi0!6uf+^{TVIefNGmRYyf)ujYW)E>^v+u71v6*mG@9#>mE|`lkVvr_d%N1l=kA zkzM~wju2?TWIpg+`pp=3njiNsS31poCK0ClsbSOK-@jvqH3#Rf1d#X*z6$ zKuoS&v73u zEq#%-xATTXcw{84h=_NI)um(>oyx}ViNYsroi{rGVXg{ULuONZSGTo!R$T$-Z%*=WvR>vf-7qOyA7!m6EJn^U5yfhD zTY?ZOZIoFJurRF`mqv+sUnI5F*0u?=UWLj{xhqlBm@H*=p{w57M4vw8v{~`Qmbgty zc^>ZD#q)V;Lg1|!Z>x8ko6VRUpLcKFRU%W(R#EiW<7?ji@sAF)+EAzN@^BAnw5<;o1UAZ1!P}WU&l}^9wfM1Y>APIsIF#S_TB}rH0tU6 zq(~{Y%=DT-iejbHl1kB3Yi@9PdFA}1*FN^(+h$Ld>lxKE1QqmscAS=5uGPc~gC>|> z+IR7D5Kg5?Nr`L!{AG2a#8+-5ZH{=Vw+r@3pIuxXd37(eKG;?L^y$-XJN+iAC}U=$ zSQ-BJ-ZjQDiRoKgE<)eZUQ(M4yozpWX;CRVN!1|{CZhJ0JMw%+diork<4M(mce>u= zM9oZ}moGARz4JfQgL1Y(b^)BX!|KFoPj-`F;sZy88zUZTF@3$l9Dq4lI=vx?$_rw{ zF`T0;76Uoam|aqlgCy3emd@^WVN`8Cnl>dq-f4Lxx82w?+s~WwYsrP;%t&UPcaRHW zy$V7+_V+H3kTUNQTPxZRM==o9 z5hpahy>8W8Rj%nDFrOlMzPTs#$A`BM($dm2N@!@Q!TIq@_bEMMBH!eiy%> zHXOTNM$g^M6&+b?(}s~RU+&57nHFMRH5f{S%njP0J16K99~dW>S5(X;FOf8xn)I~H zcK4*3DFe}vt5>F$A)g3`oL1AgtGvA2>|;_bg@Ey^DKBgqJ|Tt2FjkU}b8TaBSDA8o znL2A`^st4TQw61FRo#_ela{NyzvE`oLRQ@~Ee4NbX{SVvB?JoDt;bb6>x&j8(Xy!C zT+%=ONUHV2XIMoC_w5M{F|lsW=Vv9#DAq*jDHTmIJ8TMV!swkF>*{FQ4_>@@0jRKQ z%2h~Ekbn@gvp$PLv!avgUS6F0E^)PbX(wL-e8l45ZiZ$+K=V?f0xu(@^o!~9_E_2b z&!hS+BWv|vQ`gmhnHkgb&cY5mYYj+gC7TeQ8=Fx3))C?!#d}8?*+`?>=* zLb7kE^+?WDAu4sqvDOym$LCQm!fsBxwv3A0wk;?CXf7607akgy!~3j;#rqJC=(4b2 zAAr9R^7!d#h-?#cbJ;VQnRiZzmhqaWwoiHFCHCd_0f6E@jCEawytdkEFnjGdNmFxk zbC)wVTb1C9#oK9?IxjD;ihSjp`&;{)ycY&?%I zq~zox!Nq&+9h`Jr+;~lMp-L&X)QZbKH9D>CS zlsdi~IsExq+fzqJ2UF0!FD27L=|#HkzGb+%%n{q_y7ZV*Gv!&fYHs$Z^HPszJipFz z+NBqMet!7GSKnC=S6X13B4YOYnPm4KS5XYx1|ZmrV$?B8^{}O-#lz&jZX@ss8vPpE zSDiP1b~Nwr&ifS@VsbRr&S@#dy?vYMzc$SXE*Y_{YO4YB(L{G_-;?96!sl1=kDqi{ z{c?J^H}ehTpe0_5@_^QrG&QrYFV8=q{UjQB?l`hPmII$oj3Nd*{~{xy>+IkHv02DFcG?4_i)Dfk&x8)bf zs*?>K(paNndvi44%#h3E#^7Z$OdzR0x~*!0+kRoIV0^pHV10G9@Tr|~kp@r#;Li>- z$h*@g=PWpk7$H7~t}hALSRA7F)fz;i`7wnpTSMiyurBMP8Yafk5iz0H0XvmgjOvCQ zOc!X2+B`%bDrd;k3kYakyXMMz?XcZ2WYWB{clmoq?cKnjppxuOso7aGe(wX1ga#JI zmgUsUY;_W~%T@TuyZHXozChi5^$5H>#%b(lwEvkJWvDdUWBM~{=o#*9+ zXn?bk)?ZHysMOkVtf(m(5HBv;$%&2CLK{|s2OHtBLJ*i{@#7~WRJ#en`c%Vq^?0u!%sl>8-i3AZ!20e+_u)tey2%clFWcl1r(X zHDT`vh=>rbu)<+JJ^3!a#YT?C-0o^Kn7=`=OQ}F zteXmG*6wr3^fXWhnzy{sp<#JC?hhA7H#6dN;v8*aOL-UL`cf1@ACaKpF!^orAa@9updBLpe<4 zbuWTZy*EARvVGg+Ty{@!kH>BXb;j4>;qml#Cp$;m>Zw<}v!j%@vb13g?(WmO5>2O5 z^3bg6Z7iGvUSl%awbIz{=9$Uv89so5ar&Dx@wR3P#!I)#L=lB+1iDUAVr_&SG+}I5NnABA~ zxQXx(A4}gFsi~=@V`?t@W~=u#@9&s3O*CdJrDf0Yn(Ff*->C#vG4b>J&cYh&YFvryP7l3mzV5Ic z)CZi+2P2tx^gn&^XrEn5;4I<-_XhmFd2f~f!EW93+{)H@J3G6kHBPNEOGP!cNQ$Ta z8udW@u6o-a3RM)X=$H@Wl+((_B|$nODYhm2@*Q?~la7a04RUwhNe|~y&kzRn4Hj$* zWZILuUWY}$A&0;Z0|l zjoDtCl5BDf9-aJidbNd1LZU(2kc^PeleLV*3(H;25|{qsyd} zLU#Ayb0O}K)?ASYDg*7aPB5)*go_;|2?fs!(9 z4&#>_Y&B9WDdJJ19U>gL)>6|3S)@skuxf+cCgXc4QWV-mE>X=+st|~F^)b(aGUwN# z&+Z&n7Qm?jVhSkg^hxQc87!~(gZP__oXUDF2`OzP!usTuY&8K>ty*83QSZ@#5)X|w4r5kU)?xeQ>KH-WcV43<%8+;QKOXa` zuMv^+-gmrx^=g7Y#j^h;JryuiUDvGq-W=WBFyQ0)_#}Y>?)V{ghI<`zE%4Ps(zCK* zT6&Y-iK2GljIvRRG*w1*26}pWlk@(}vD^zK>IkjhELR<>lJDi||McmMpRJ+e#BxnU zczEjFlO%Kd1AdTf> zM?p#qmsR%n&(G$fh%Sf6#AE|misaPQ?0b%45@MGoH!evj)*N2i+2}-z9QM32>%Krr zn)dP%>)g-eyA=CvgLc4`?|voT4vy$st{tfr&94@gsj>c+)EGuPx1ObpkU%faH*lMM z`ki^(x}dPIhMV#eo zEWag{jacj}}ME+a*1%hQ@|=Ec{k zStx?x;J5z%*^-C(7GIN|>`aV@a8H!AAV%w&nH z`yor_UH3lgx~q0}D*6xHzQKw{dr=4*q>i}Rr{1}9=R)uMQ@X#7IL*N2t8S1 z&1!0ivt7Bd*=neCu``!uisVhCLbFetm@P;MK>>lGss{xZ$S`vIi;T_I1#B2+Kb?RQ zE<@qnlf#4QLFv;4_K&?1+@{D?&t#_)8M;Ev9j67bnexmfsSk<(C_jhE&o7e*DUIPa zVn=48FygKxCQk)dKyNShiL#jA~Vm-_NNW;iS zBj$xhS4@xaxNUVxoEh5QS)I;55b8mx>ru&W-y8K>3layUjg%E2#~I!nN{QOV7RA6y zc2ZX48%hLuObTSFSXD+jcVs34?Slv8D#ulzn`{aG)45CdZB+$2Rt4X;zsg3&-=_W$)~? zv$a)xeE+$d(5TjTC>n<6e|cwwQX{C@wO%~%+;sF5X1&CkKL0cMGh|X-Sz$NVhHR@G zw}!LdG$`59-{g8#wJaPja4E`SYHWLLy2P~hm};hiC?sd*F1w4oxl8ZZw(wOrrIlLx z!?v9U%Iz~KF8#c6OiT6c?x*VJQS4AGV;T4I|A*F}EaDbK%n$Fjmt4GH)`_*6#?22Y zDJeVXF4j?^CoBR${HX2y^h7r>s&Djmw+lhdYUGqL;U@!|CNZn(*Lf;olIY+K^QrIe z6x18KSkC-_bvIGY`ZVR_ zHcJ=Re|-k8tnkxQ-_S=Ubd}KrwSVTw2$QnvolN<`QG_Xd7kizd+@iPt_fNQDgAb#& zSNac3C8HiQ3rpTp1_plQEBMK{-)kB*?;6=Q4%l7OsXop)J3q5Fr!%>8H32uS$EcN` z>neWl*#P=7Lg5G!P_>5ct}chIj%$F$GCr>D73^*b4x%7e@ekBpUBgzIabF^>O(GJA zae}%0UK1?LDjC??((4x?bm7O3BxXsoP`k)WoEErx$G2nP%^hdHNrr}phQ8}hY*PXK zhH7Gog<5k%g9Is~YSY>kfl`}ppl0k#+}+(#yD1?fd+%O8*Y9{nv?nmIT2Y?yGDiKe zYGu>aRYx`k?!lbKo9$-zo`)F6OR8MbYgfv-fLq}850#ZDwZ(y*`H#P$Hh=>|XxtKF z67^v>>?C+%Nhz889pJo+y+;#W6P7mwmgqKO&iLv75t_;@IXe&<<PYU0syK&3#Uk zilolixacX_uP1-2LYbV=jkH~P*PEx2g0<*t!nDe2L~B&IuvdSycXd;v3qJRC9^2SbpLX`$$nwhDc=N<$KqN2#@qZCtOF>2JykWzN-(yDMi6@jC zltRPu$qA&o0;#+e3=Dp>GAvg_G?V+mv#={!F6H0@s})AI+dJq+go)vs>dxRFwPr1dGwY-y3P8Z4XyNNwO&QEJtn z<31=p)U$R`wfqcznfCp@hKwIHlTWp71lOM<*7eKxQ#?yLd_l2O)?Jy!e6`V~&S+95 zWS?oSYG}becr^P8<@VAqX!j9o3#h#;fU5V0|CtEel^?QvCJG9UC02d98yjhtTO%s< z-zQfV`Nm&y5WJy$?z_I%RGs!~YB3eNGEJ7Dj$sYux!>oi#|t9+{iqTHqu_gqhU2Bv zdM_KXy$|E~wMrkne*5jo+`-QqNW#d$q8kpy%z;vy{PPXL^B)g9D;(&T8~R;cU9&nV zof-BGDu4)LqQ7}zVW8SQE5KKY&!avmwlm3=d!KY+Lf@?N3h`=7t#|9o@+4RGOBdGU zecWhxQ$!6dwls!6Hy5$}niZbq8}Hjb6(W1jeed4sS!>KtDKcgjE4InMw1W;eIjoG~ z!q`{qb;LUlr>W(B#-wTSVsca~Tb{zV5#rN)s*N?$mi6BLHr+z|3JyeH=d^S=u8`r= zHBR*hXm`BsL2){g2XZ&mX4Ll9&V%ZF#iGco3j-y{Lryc(cs*`!3?-Y2d7!4vWi9UK zhVPRrNtZkjgfuZ6kY|6^7|6mU%WKvPBGdBw;-KGnW^kvWr|pVG$2!l;!+ z8@7?IU1gNWed^cy6|P)$U0?@2jaCVWnAlCZCn9-2M4Q`j2&6{6opoJHXnw^3rRDUKx2ynRw+9pI~0>J9V-wPF^{CcJwhT z52u#Rl@|qbYj5*2Ftj(+pFLF%ITDs=g|!M9z88GzE78p|yM5fm0VYx_M@)b=5Z8Mj z=0|g7DAta>&%~3`hUtt`^@-kwUpyDy*np+E`j6Ff1i`l`P%2@Kc0ytD3EeB+#;e%% z&92B)u3Tvu%=4523WeOl>kJhd!lPBbFJoLkm|EDi(O$P=tk9s0%84SDE+io9ZzTV< zSxFvrFrj`cQ5yDFV!F+i&3EbOkd3yQes3oY8uu~P6UpQEu4FdO)WLd)7?z@vF_!%CX;Cmoo_UP ze+?5)ozb~=Y$W{UB+rF#YTlGuh6~|#ktZuD2&Fh#59Fa;|)exML!3$==baQ z#Z7baemN1om7=^iYf0{mvFjl?WYpvQ_mOop8lEiQ8o#Ft<{#DdjU>vor7WA>PO5cv z7EALVwBs|uQ7GY^G|1~KRC{9HlvSi5E?;;`Dbj$|$gJ7lAuUEmZJNN|zU#TZUaV#)P79{b zcnZJy8aTA$C{?w)&L*w=@kv}o8lmCl9HevQetFO0rb#!}HyM_cj#fen-e2%M(As=y zHapGsQZdgpRa=ez6C0ZaybSL`V=J~)^dtZLwFbBOB-kFbm6H?A%`T^b8$x1znl3&XGox<|#yQ03g_;5DBv=+_(C0_#pg%xVX4k6mESL zrv4y&H)QYd=mpoWKYYZT!dF}(C$vmG_Ure@+?8EN@=d(kjW_;FFL4SV_i((+_t-Y(ChG3Og>3=+xHMB3gun=CQZZ0B9l=UNo!qJ@HE`MM!=I(=V!2vny*t=>8cE3F+`ju?QmJ&Z!sO{dttljuI#;(^6K01<70r8dvVd+h2bCbG z7mGmlvEFf)gr0^b?E~v>dKO4L0i{olkKd5YR>=ag<@n_MJW4S|e71DZTDj6`v8TvH z9@JQR?SgBsjOxc=a4Pw_LjItno6Sg616q}OcA_!NZS2WONE{r6Hoh4WoH^6|iky?0 zesq`e?Ac^uN`6F=6T|OlPCs;{Egq7oyGqD^>z3)lw*nBQMuKWHgYq zX;Kj|K-$(s5k2K}nNAR!F^@lkp=Km{g~rCl>bg#nf?7*PI+6)_t;A*J#}D+vIs2s{ zKJ4P4IwDzB-EE5(;D+TVx+?epM40O0;?E|(AsVODRBA*LCohBz|MBAo_2knJ+r!Jv zhOKYe?+UrC)0LmWX?t!Lyk~tOn&3J`4hlJQFd5AckCTPB#*Vkf@hQg(IJXsfmst-x zG*CJtdQ1@WbphqT4XQ^;B|yrcXH9lru2u*AmI6pgnG*$od=V59qLD;|gCO%=s#=o+ z4FbKOpmylQ;$rU1%!~p0N9cJnGSl*tz??*GG}l61wwgu~q|gJURuglFrH05_`BLZ8luFuwdC5!!5lz z?3@T(=lZ8JtO#p8>aiB&wD2u?X2!I{dRSBBaIYxA`@j{nD%oG=s}kG|Iii{JSw%!v z%!`a$3#I~ik$P>D8jM$7Pf?i3^1z*nybxot+YLHrsXex5=u znnR4W>ZkAC{yz450M)rYoXh@3=38rrh(itQUv<7^H?*U)=*@{%%h%N*P{n!bq#S+q zBL)5cPl?=_L0hECuOU%hU(Kik#*(Ik`|{h-%9;A_q4Hm+BT-&g&FBfM_^;Py$o)R% zHnjzluHn{I{RaC{@c6d^@Ba`{{t@+kMYF{_Ee&Z5ISv(4@LFpR(BK}6C#IIEuW%i& z`OFh~++VC!nP~oM8u*vikDxl7?1QS&qetO?cUgc6rAg@I$$&1Fqs3~uMs}~<#H1n6 z6}K&@H*b%=g(vf#F`@8>=|_2`4IV<6?omed-OCaN>n7P)&hMXw;Nrt%7x{nI)N9NAY;i?=N-azcr5vFJMiil>ZkrP*gT;Lj3EY zk4w%;%3K|?=TILsL??(1ginZjQhMr+exaQ-*K^WRc!BZhddNi~U5PuoJ{$oi|5L%G z!V62`b5m2{sMhwzWa?+76s-CD|C-3yHwrHrc9|!D+Ua3Be*qZ0@YDafkN;&czm_fN z27CDcFf~y##(yGkkmnkWy1T|VvJ-U>qW(_P>Ffbn>~V5Cvz?!sQaN{5>&v4@QyM3m zMBI1JZAasBjUlD`G73xNLG}L_#&`1nP?936+BWa%3%f(ZUW{>}+1I{{n^;l~!yEng zObdIL6+3h9Ig1|IV-v?w*&Yw?|D;ky7L9{->P}uM8}ChS2j?~nnulGrAgVBDz=Lve zZRl1io#$q{c28m!dHI0*mg)!NecV^yc2zd79fLh|J?VFvzWi7f8*fIHU+wkv;aEGx zU)y=}$Qpf^4d~f%AJX@edQ|_XLJX9@^M-Li;bT1l7yo4--2aR{^Ip0Vb0u?Thi60Y zPsT{J(0+AK_b94kA9s#?R$u+sa{cQt{?{h{3rLbmvu)K~()&6%IJkO~iAe_fEmXjE z-?W{-d>I82bELN;biI{#G!uXu=rm4T#cdp(0^9Bn!R_8$tx3>0frNbqx&#oVGKjre z?|5!B{bU98^1;D@`Cw`8-rgRR-3bJb_9jWct|I&!^dUgJ3w-`u0U!>7XCRli0HkEz zllca;n0Yr7T&?+fIdSGm=TgJ@u$1q~!pl9HVe)DR5u+j0C%Kzs;R1~{r3S9WwT=)wXW>&J{t z+7kd+R$jPzH4oYYkP6ApYBQ^9niOe?S+_W+Nm~!VBU(vl;)pYDd7BJPPT3M6S1Uln zU1E-504#MM#Py-n!(O|KngCK08mMQXK+_$^XU}CbBIxSoHrIGFu}90MIt>OP84B{} z;zy0=WD|vTkU$RcmqtL~YS{x_BO{}Kn$zS8@clr#A4Ux@8398n;z6MyAt9jAN_lep zBvRBx%1BVLnB;^|4L~P?nx27SuA*M2z`!4*>lx5rbwBi~_ns{xqCZFIG}j{uC>?nt z(%b{xS}2fR^V((6eO_3Urar`N(1)}O76ygT{Q&Z-M&L88t*t{-C!3>?-PxIp(gU>> zGwjIsbQ`iC5QVWhq9ADmEt|(~(ZeI?Ybjfhmqx%p9 z-SSY3jGgPt&wu^;4>1Xe>3&`=E|tZBlJ3{n?oQ0kYRnk63vI?37#KipfE6kobYP2+ zGO(0d-~uY1HTVApdYsryn}DXf=6UEmnC_o$icfb2tdaD(!yq4**NmxsR;$7d>ECYu ze}3Zr9cYWs;00=d2;iT@@d7^?bYGd+f)f6+yZb5*DfysC8Q77wE!)1Mz@GH`$}3Ua zDE=?N#%m+>A7IJh#+0{{fB*gWzr*vSFX-=_f8m92zJH>M!W(14Ho$Vp$;oiy-<#I}^c8$auxtRFF$qD2e4^ooeC9bHc_^;gmx~RWAqUm zdd&90xRnArxY|~Q85pEM67RY}L`bNjqS6yY{DOlCiu911y#Y;1RaF(zst6_=<)nd| z>W3zxlYY<%Sumh70bm`{!q%dEcq@x;l}pOqcdw9Ks?ex;VsbJKX=SpXZbOP)u#QPZ zChd_s(;S_E@BePDLf$*UvBC`F^;lqN1W^wOa|4IeXwsk(RDGL+B6#f`fsLO##>k0}WhI zy-ogjxONQFez7VCTzfWT0#j}AjL?`%Z*qF&>FM)G@~r2bMi&Nw2x;mRyLazv8_Qv0 z7&MiF*LE1RDg!^4@jK&Da;s}6zs!&d?S&`^A+A@!WxYjj{Wyc2?UsQ4Gw8E%1BL|y z)-}&^!6*z=Ezo#Lfax>k%7%WRI&DR3XgZ`3#x2}!4fRz0_s+{Bk$vFalz$02O}HrF z;jG}<=j*eNTl+F5y8gnH9}kP1r)?E6yHpLMe;)gh=X0!(g2mL#S1ZrVRa#jqzdT~2 zyRgb=xT@4etGbV;Qlm_}u&%f&Vp3C$B_c-P?d7r1V=NzhqW%-BExLTo%y=YjnGcUZ)tzAlTyLuejw;5(JCoh1`r_#De7pHK(RBK6ZzcD4 zug~KG?zL8D5ET72i-j;};-^zxDv@MRPG=2M1_}_nmySG@xH4EWc5-a~T(3^jAW6&R)eh!*G zdz>hp;II~c!d5wvq86~YGkJTFZTpJf@3+#TDRx!{B7!<_#kG%aZ>4Y@VR*N${g0>2 zv7P;8<5B%aG_B39c8G~Z!MU}^#nVTqXny?pUvBv{-^5wJ+qUI=e1EUS)U8Ksw|ER3 z%IDjR6EFi`uorXctML7YQZMQ<=jc4s$(moX{(G9ick%tM1^MRc+{rcjj`@_FNQFCy zOn&x%r33$7x&PXezk|}))4yTh%|~Um`Z3>~EuzLQxWwaj{7RS(zMs4D#%xUuI-{C? zH%BM%u9oBaW#(^(PujGF4Hu8NLVaVWON4p+J3(rc`}Hodk304X*9x%*Xp&-Hqr}2| zvf^6@oJW;gypx8`wfc`!lFr3GiUhoj(*kUsKH1t{6r7>iWAm^|{wR?ea~aCfxvlEB zS}qs*N`mNOjZQH!H*XbIL9z=Sz8{+R%+GKzM6G#;Iafn7VsdKY!BJv_7YkpJbnlia zeegNrUPtG$zp_o9hU>Ls>0z-JMne~M(fqNlZ7!qHpKG~3FAu1nQO=Ha<9d@38vx^_ z?zcG_gHI^aIYR54G{_{{D2UU^yO=ik&l4Y|{vwr?ThHlX&f;dzMsszZ-EBr2bX*ER z7dnySwj`4_fDk{R1T8BoNTF3p7%-*SpmSqj#A*>&^TSh+y%;SuRHnMDNxnI@eV0Ef zLdl^2eQz1xFqeU(`H#-jUft<<6DRU^b~@5$zMEpN9#Y^On-68teYS7;s(M8s<-A5z z$@=W@4Sh}XRdX}EPknJ;PW+W6OrT9k#bF^jTj#?})B7w>9_9zUr>u{p*3j4NJbY-*W@z_!BK?B>nDK zU%!xZBEeD6(Yv8wHFx@k+j{(Aiq6?+-Hl_6D;Em@zLf#77Am;Ch$I5+zEwi7v_ z)b$I+cdR)c?kBiuu<1`W{1wu0B2QYM`AkmCiPhzcihzbaDalUW}k?AyAwsaH2$1!rIU~ z9Gcod<*gkewDG_{Wc2+*zpCo$h`2b*7k0SQtbYS+nMI@U9`Ik#@L2V3aj?u{rXx9q z?V&Glx-A@DqM}bU>I)>=g?Dq{28(o7^wu4OX5QbyavOX(H90k9{^{vyx1Y5qb9Juc zW^MTzh6zpI&BMbwIu#Cz3JR~0tfG2*Dt2{!J)9Z=LJ+PAK*WiV{D8HkLXnniK+%^Q zW``@wkY9jkZU*f)C`+OsrX%G9fb7z|!wz9U$aCv|#M|GQ&ks$7mRF08BvFD>rxbwV z#dgb!X#Akpf)nC2N>7)Geg^}&972)vmC|`A8K`VsNL*FT&9kq1{1gR1YgYe?d}ks| zejQ%=g$bdt83WaaYHY6-i1QSbltND=lo;I;6Qif4{i`zI0JlNR0dc@&KbRooj@(j! z+|ED^dV0=!M`+wvf-Z4gw^z6pO~YHALy z&gg}O;&xj%Mea&*+n$WVj<_!)#h{U<@ z2I<>=_!%GRv>z<9iK2$9KJEaUp#sZ;(D36beH|SN{?OM0MXH-}aS9+$MhXqc?J9g; zfE^L=3@fe&ZIYcLhuiW{4q>={{SFWxF?{yv$gLmN!~C*ww>lrPs410M>*Q*eE6EFQ zr*16tr$7QhdF%0Qki?sRd;c(^-VG97?3eTMrhvhvWMp#UeUA<_Cv3Rz=8%2GGJ1C% zj9mxBU45{Q`N-`t(D<(cLTsS01tC+BhnLi0@Ni!O6R2u~uJ9@K={U#3#Sf`~O&bRX zD_d5oA?tu%^~IrbWoYdmsBlCHAMV)$B9(`z{!CChH6C`FNIRYX8Lj_-}CYFF^Qz0Cd*r)@0|;E5|zC;?fnsJ?I-nm~ZXZuU`l9 zK0jO;z^(nqi|qg9zc~O;`cy+E@Q$QicQ%)8HVSadfak8BrvDQ% zU$@B!buPnt!tLE%=}>Tu?@SUs)er_tTPN2U?}z8+aE92+i{#7lakms94630T`uE|c zm2V%iERt+ikJsq#rDiM6k%(u;6{r|=Vdj>1N+b9uz=1E zkZLhxznNg5jI6Bkcb^fyfg5;K=U3pWhPeaW_$(D8d~4xup4sy01jdrRHi2aoUEPG0 z1Gv=Z*M%gI$lEzP_oajgXOE4INi~Q-84i4e5Hy=Gh=}Oj5PzirP3hE<;0||YQbI1A zKYs%qX0&g}x$<-WS6T{yYGD!+JEp$NLl>r)bbjt0-An@@7 zSKCcWA-^C#0WM|=hn5$(c1H(j#R!DRGlR2_l>R^$4LPL6{-TX{SgclriGe^>c=!d+ zwN_rF{d;I=sMrl_PDIXe4~p)^PG&f}psSyF#K_613|A{b9xt8c2qBbnivzr0{$cX!dJ6+PtUrnSUFm6sQp!=Wz$Myv_sD-y1OV~B`~%79Nwg(BA= z@s7CMh0CWDpc3Uc>bYqKG^S8)6Yt!q=WuLh-Dix)R!k366bZQ=zsUS4%bnNg2p0WTqb6t;#v!G`VDtp^B( z=mlB8cSBx~gU4D|NCKzuTICk$R$k>&SBS^~oup=&b$opYmCXwmX`8>6+kYbfqPgwq2})}*F$d+g+rJ=B5ysuBvn?rr(LqW zFDhBe47Yp1OPKa`EO$ zi%95(in=tDMPFV#e7#xKC~)WJ<^h+$#?ni=%_IkKS@bd?q%{&50BwMKw&+5~i^3%n zK-Xue=1f5)>a4ryJ3{Y_ZtVXxi)B_}Q1pp2#Y{pCgl~)Haxo_2Q z*}1+EtvpR6QU?HINA;}2(Vj+mYwPPO zaBWZ4;}w84JZ22Gz(j*cL&cyUj-*V*pHcvVq0QVX$9CfD8^mlQnKYDU4dX@Oh&X&E z;zcfQF?!}JhH&jQdy}jca90rod&61+oJlG~GMw-u5&Y?cs2+e~CX0e%ICN`hfchcw zR~f))D#>>e7Z%&cU^3S#T~HkgiR*<0%GNjJ$TT?d zZ1@h&L$(2k;6OHpOA&`H0#GKKAR%JkJe{@o~UM*$`CB0Nqdsd$_zP$SZ&^BP2xzX^nZ zzhH*XzZkB6Mch}@WK}LJ@3`mjWPVIfoBYMuT=8WmBRqBLy)mVaB+M=MuQ+#CpW&$b z_kV#^zUHpVroMgq7SM<_ph~q534M765!GI}*=17Fp71;Oo1BIW(Z0)KlW|r@k3QRj zMb2^WR@dc%2%;@OB-D${5ZVYBdEDewbg@STvc3K zCY}3(mXH;&_qtuU$l=EnGUT(V|Iv~ca zUM6~=dey#7xiYAX@$z_2r$esCs1xh6V{34cPr*O-@c)N$WAak{ahLs4xChf`m&5il z=Uf4uloiIyH}(zWg6hjuXD`|Jyw9p^4L+h^2Mt#Af9khv+x5}hd_>20EoBIs&elM- zG1b5t=D@@w`ybX znzxQRbMLGAzIs*9s=3TuwfmfX_St){|N5`rvdru~ZBf_ow4*qSig)drgO+G7MHDJn zXVR@dE)JkV8Uc&cx3ApC+CJSnyjRSF#@@F6b5O5_M^o;m`2Nh6TC1Sh2LgV4NyVgs z#V4Q>uCDaNDyPGcl0K=p2sCOm{Em^5cZ)f$)S}5&^+WKG;qmfBVT5!^mujQZ5 z`wpLXzwMHrh~5uvfBpd{6@l!>a;mt(aI)I!=e;KuR$nrbdQ|^2tNLkP-WBYDZ<4zt z+ZQ`R$*AH+R_Ib8keFl}p6KX!`&;zGvg3!DAiNa8|DLNUvWGnE(|$UU*qqolp~MYO zq*f-$3=m=9^lv;}Qe(FN?peP*E!1ot0?(Se%H$5^Cp8()!9O+3Y(TUhGC?yf|6^ zf%zYq?4b;kJ;_0dS1ChvG=wx8xPI_f(?$nT3Ik$@T2 zF;o@(mnI+2D9SE>j*>c6vNmavss7ItRG|7FC9wZQQX0E79#t74EadG1^*bED<6&<3L2lb9RBXUYyVPyG1t_@C9)>@8?eFafTAqW*J3SAV4a2?zg| zb-I}K(g4KQD9&{o8+hLLdHXSx3?VnA>(jO@mDRiO)YigiE5nv#(4i zeCQ2r8VBKs*i7tPthd4|+B8~;86;zH?dr{&UGRrPfs}&$;d}eamxN#an2Ak`#Vi}G zE8sdre{wsgn9wA8|5vKvEo?Wx*EY+i>lYNb*H0<9t&{)y$^EuNRP}FC2&m+q0feq+ z_4DUDh$?0vVhHDv(#=6rFj3Ac;t_-Ed|o?jwP?d^oMyD+kG~;_Xu3AlhD|^~rU;1i zB%KH%0U)BlKB2G5cD|nr$zUM?4Tw}Sw3x;~da~ok0U)$yf@kUXgOjUf*`d#|o%(m$ zIz;bmbfSL!Y`cArT!ZO?cFw~WaIyzQgV(J82EtvfaNfYar#^Dye93n&>h%Gm7#8rS z!ZD`<&Ur{JX@$$y99()I5auW@USxqsTtS#iN30tK(uEpXS*`Dmn4A?R?Kv)dNSSW$ zAo8=vk6}GfCHgMh$K=uyCnQ|ailAanz`yJ1>0Q^W)P^rrgv_69w=5qE9HM?gxvu)g z{D2Z73Bo}bD)Q7!=Klu;ao#CH39`zeF);uM+nj-@2dTczs@T*-0#3#6Xby|w@1*E1 zppj++jc*n-o$#WfB1Aod+-N#P%9=0$j4WWeAQxT_yeBHk`?TY76u(Vp2m0ljg z9tB%3HUV5SzeGDmG#r&fs;a6G^1%D|>F_8KeJacq{b!h=KlccK52C?v^);1CJwt>)A? z5|e`&1mZ(!Z~*}+vm4|-9?NfcfW;~b>ZFRJV=<#KoQM$v4=uPzpmucz968Dm2DlTq zjZVJiNT-Y@@; z)wnqFSOEMuX}&*gCuUO?&Z(C*G4TYd5Ak3m0>i9qX0MKxIm|Hq8whhb4dxRPA%dV_ zH)7WZ=+Kq~D~P`U+@m>#=JE}rh@sR|Q21D0E(D4M3kZtv7GPm2(YnlYW$IOpUZDD& z+WG7b7xNAT1|%wjxd|ukE;z1BWk3v624X*=Z7z(I^*O$+xB{_dAbbSiH&a1*gcn8* zEUy_iwDS$X+1&x^xE)Z3kpaa18ex#;;Z4ctQWu-n5^~=y2GjN4WwM}Gap8Jxpvod12hZ&Bl*eC7) z3pI#w1Qro&0zE=s@Pa@Eo*{UrnrY-u|Gs3^ABJsFU|t4*Pr;X5(B^;&YMfZOx7O)V zcM0V_>{tdvf%p@)ee!b=ZKoK6Ts+x*U)27O?CG3GwNk?|d=|P9JH@bUgUS(kvk#Fk zzFYwD7zY}lEavw!B*{VFvioxR_qUR?NM6-*bO zxoy}dS9WpJX=R7aIc^*3G$P7{HQPpi{8_7 z&3;a|V@3~DoEyHD(zWoTovtT3q=I*l^8a4MZSeEm=!N%1>9%E`yRE_1X8WG`~kgx4|na{ReYUL2OnP8+< zcFtSu*M0LBu2?1KFWW{Iw7@of>hfhF6m9_Ipj6>(4nkv4E_GJ$Je@cG+Dr zr@X>#Vyz)u-bPAFMj0yG2O0{P>qPI4?`a-C@+{GNCMGg3ow&Wt#=;o49K+94PeH_E zikup*duOf(+$WFl(AiM9PFM?iiMe|y`5!NU!@S_)a?IT`!cR`FD;O8-?KFisr7H9pJCvll)cYAD&Q^gan+P|1x;yZgMLC08n-uAz>Fi|YY^e2~j( zF0ldfXy)#V&}D#jcF^gK`n-DesxjTYjVKjk%^}9ZGA+fEwAq12W;J$+so%M5*yV+uz*{ z3wuTgk{bx*RY% zQjb66&**s;ad#i{5VCpY0Z$hlDJZ*YY(Qilb(f#j~;x1%IW@GNH%O57vjG{e@ zP!aSM%TtDI$N=eK&jRyI zfY#4NpJqsNjhmdA(YcCwah!}y(NZ!6;dGp%F$FHq)i-BtGeADTZkh!7mQ`=0u397$z)z5n|?xpGt zZ)u^qLm=!0OoZL9Ir$1zpj$vHe2>j2+8$Oy8r?9-ePZ3St zLBpZPdg-JTMNG>w?nr{LTEOehX^9q^NZMTaLgGcn0pfIrk4?B#)Rt=6pi$s`@&PuN znb=si#cNg=SMMELqo0>1&7L_;5%-j};o$z)%a?3p9rdS=_mX7$PM$stkB>OszX98(*Kik`-D+)X2<%hy9{dD} zn-=6MRL3=0&#^1wEIVF345YQDDcpo1*qRPZ{eD?^;N{tNS%o}@3iQKrEZ72o}=_UznTDSBhknLS5Ia0RH1=XG2BaU;S6MtiF_ z2F6}IvP0{By7(Q7f@PCOKO_w>PS?x@V+bW$PJqh-#8THPWDDF=*`-%z zuOV+6uq;!Tg&(ZDzkd6+7v?Ch)9RBo)J5ut%^@60&aa7|$fsuL&|!FbX`nNFf7ia> zp3&_yOW#m!6zCKokogVfc15WhQ5kn5@Z{n#YVn4& z6r>(vaioRHphn`d>OS0l+<_e33Q|iYjuW@DpY-+)=auL#?f9TE+czg)S3^X~1}T^> z<3nN~n8(9G1Xv`Wde+ZCwp2{qPDQ!z8f!uaY=DzCCLP1haWwfsmlG$!0AjFxrl zb@l4KyQVv9K^|9WxFpH>Z<)H2qTjCYD-Q&DtOx4n*^{eGzrO1NlbJ^1xM~na-NX7| z#f1LFs{v=>sSY}9sDjco#d?B-rg?jQEwXU-V#UBdFYlI_?)VB4`GT4GUAY^CIfzc- z>8W#Ka%X)ZIVI~sqFny*A%$I5d(7r^l?Wa1Goj|;Y85kN*OErc*kbq}F0l%j#Ip+D z-|vq}+iZt)2ml^C2N&qIP8?1&xx|lLtcZt6RxZO4crfDRxDO>5)ByGSN>;0#tWmP3 zBMP8o2x8{-PJ@D|bg{m$N;~ZZ75S9zn%f9uO`q4{wB&5tBLS zdDBL)v?y0`mzLWHT!Y};>Absayhn3U^RbhP9w(upQn>&#SZHe0OUv4w;cdo!jvX7J zt3F=#UUu*Ut+iS2?s4q=w9V7+vpjCr0-3`blWwHw^}u)-{!rEJrDIx82usWj`rXwJ zVYKTR4vye)om;+d(b%ssUvmvLNFJ_HmGE_q2L(jr7;H~pVfg{+O|awzSKWag2#vM1 zqS@s{FZ57tX&f9F$h+L*BbS&QtPE#Ex*f$RVo=&vbP6_gEiqgit#OO$WUr~Zdi
  • uxE(%uu=`lau*m1MAHfIZQFXCjGg2jiTZ9%Wr6{pA>e8zWsQ$DuyOjZT{+o^%S*+L8i}gB_#uG zLp1g#hD#UEk4;w}^UlAv*sa(yqBF9Qt;?CF;-Px=t@3s(5ea)C+nf$vPm~>19;2JT ze(i+}^u~H#dzXuSoPji|z-(46vB8g)TRi*g2aQ>mxS72hAlgKR8F#ai9cJowfi(z) zo26^IC#?M7_J!ffWg1L@6?{~wYwEJlOY{m;xEjN`kQsW3dLxB4-mLL7ioU}tj>>|< z6UAnE-mLp>kH)HEq^w`*e%d!8QI3{Eb+gBn(iO37oX?B_yc@brX+67>x4=3_^Ke`M>DB;2a0)-g@o*=O0#XWeC{ z&^?vt9-qFI{+#xnl`E+ZuJ~=b`&;&XlP_X3^yk!egq2~w@4$M4WmuQ0V{YM!hh$6s zVApa!%EYflt}(t9T=5C=XNaP&y|kbvKkB|ALmS{5Kw>F= zL{WdErZ+q^I7aDliLds$Oi^-A({!okkORnrNEO<3?P3M+QbA9JG6V|>$`$fTKg!g? z56tfO2s4N;8+4(65(p;WDF2te%OJ$~~xzJmr%&V-!fCv9=W~Y_+m+j6Ew`e_ zlRRO!%e<=X&)gnB01~3Pwbj^Qr~k0Z#J_C5(b|osEagisu85Owdj(u*WJg(d$Q&&> z3Wrg+AbTb~zIw)|^5XWXb}A_IV_O*I*B!QBs zw&6y}H%-@-WRPUBgv+@RC{uN=nKH zxpQTw$<}@&v+(YCDU$NdhzoAFb50skvcA3yT0Ob{O*M~OCjOiQe%_8Drv0Os}sGi3%Aruu6*WdGapZuJ0V3}zH;uB=q#p) zXSN{H`y~r=V)htI#qf|`DgILpOLOR3D=E#jPdvoBxxNFp1BtSDynEGErm)jwf8lDE zn7h>{elS25S*_H+%;?hgPc_##r2uC&&Vz2VFYlgsUm9NaOv*aRTBg|9>W+G?RM7zv ztxo2T;gL?ac*k*h+O0e%zuk^3rXZ5w>4HC2TpW1y(o$w`DQo+J^caT|s#xc{r`Q}H zw)7d6od25b(HQZSU@&YO0F@*wyMzb^jVS=vNiV< zig#A*pRLY$dIZpkWDY#rJ$m>>?!2bjs(V$LH0fLV)1LdTS}N`*Yhd|XEQ+iW|15Xi z@2JArA_1!DzE=vT?CUq(^-htgwqB|J$XKw@C9@D08D6p&uEH!s#gL(Y_SyDS0;BO5 z=QQL&o!-PB-Oq56mfxT~uRc}k(K&4g8Qb7Zs9yXaf0j>e`~fuyk!V z3?bNMyhB3D`^40>tSot14ovxsUDva-rrElmPqvwNLw<|I@~`v9NU3kxac&(&1vn># zc0%GhtE^gLr!eW7@O25juWyk=`YO4p!c_j_0B7^b#kHQ!=43uZrMq!=CH}DQSHTMv za;2_lpP2{6l^JWN&zHEOO%~O5?EWHoeun-0@kr^V?=Pg}TOQmSUVY{15ql ztD^mhCr1*qeMjVdIFBD=aFz@xryt=KTQycG=9ql)-7Q<(vxv?~)Y&-UTvgu0yIIOa zLzVXe=>I6CPaN7m2ZkmuiSSEZI~olAMu<+=4)oCcmPoxH*!Fyzl*VX>{tRF^jOSxL z{H=0JsS4IUQ*;h*P1jLozl58?lZp%lwlJF7 zFEL@~I4gwa=JrCAJS9$VR!Of$C_0{d)ZB#f=aqDJi?hWSYhH(fn2;x*5Fdk+eu~=p z{SMO5x2LPvaE@h4G3P_QUN-%GF>KNea1iJZy3{)>Gn)*L_LqDvpxH ztcHYR$%&$!Q`sQID>njuq33gRJ4IJYghb*HWYrDI7Z&ugYn}~>@rGm6Cm%7uCp|Y= zHdb+bHRfcLSGZ} zFt@*+3lF-Q_{t^3q}Z1ITsv}I;RMO&MFO<0U_)7wDosn<5FZip zAEz!$R>R@f*C;VSFRfybgyehyvqAP|!hEJ}8OZf#jt?GHGo^$ZA^H4Oc-3otVz}mW z_RXTV!MZzz%dQ?M_As(l(+U81D%>g)czlDOzV^5M;hJ>cUC({{%usmZZ>-e%a2AYW zBihB4?hJSw7zm_t022i#BB`R4&Gh8a+JngL!DwkJojU~+=soyx;IklS+jo7dxt66V zBJA2hk`MLVBS{*NpMTEe>fSR*(bL*Zr&0D7$(<7!m2M>k(sxe4u4Pr0GBv3i)gQt@ z=4%R;5q_yZxr`5CI#K#Ll1>t|e;Jg3=@-Ok8mj+0KkzHbJ95awD?Ca26#0uJ?EKlY`e0!w`<+$zm05wT3O`a~`BU_cLJo}3{P{oWZzKhNTzI1V zzsv&g$A10#{5KfFPcA_P?td&@%4`=}fBQsnh0nAAhc!6N8=(Ta98`y7Ap8U@Qi>uA zL>us$=2{!%oa5tLwNTfEf~A*Qhb6IZcP2Q8M2((>uhh(tr&`8tGj9fA`# z1LXLDwJAp~>#Sa}v(9A>rT1K7OM@)(?aoNAtbu40jn=aY3z`gGH)k20Qk zT_uH;ix!DTh)g9qK;0JE{;7XmYqJ2#g0f!jxYP$Pf%mwk9Z<@evpAKazrT`}=Kv9Zq8*ksc#CKp;E-5a4 zXOM9P42sV9%p$+)Z_sxz4a}(T?C3}o-dAc85_>as5D2;s#*MP1)cX9igM9s_Ed?;#yLi!jAHZ%;=ASXS-XPCc2UMA+_{0Q3R z5f;4*5XU-WH;Unak;0$hsN#TTn9$6mQVW<~J>a+SZ#e)&2t zLkmE18A-rG$@mCR!JrLpJYstQdvG&Ui^0sjN$R(=wbb|Vt_M=a1tn#8@UN#?`oWd_ z!Hpr92_)qU)^cN*NY2n@6A6HTesK*{bMSIhfg9i|%r}Q@E0h~yy%W4HjA$S@1tA^} zC{2Twi*~bJVo(Bqz z-}-cPFeBJ~<@B!a<<^hyyVM__oSaNmTN)*}F9=*hIZ0KSjU>76S%ffFEDxikUwM(F zO@sY~&=8DK1Ji_n?R-iA<@K&4y~3x}-gGP(cBW7jhW}J0i)a8DI;;xm?!!5EtIS3- zV3N)tGY+F&?k+9aBe+z_27a17`-#^g8b3c3v>bY#cEkf_J(6=rxC&Lojcfw=NLU0LH;!I0mZG!03;xO-QW~wq= zPlm68=NQV@5Q_~0j7h6hCQ+#21kggqm^|^iYk_qg8Cr2QRmpkxlLQ5B()n-^nSydS zeq*a4c?KL6*o3+m*SKd|MF0v`QrZ|y&%Usr#m<<+#56vh zdS?>>9q__m+^AV=XMtJdUMT{c=*~g#rY9U>W`%DbvDJqL)N9dT^+m=RrAHS<9CxYC z$H?G*?ksVB(gVW+0)~NC5^1-)Q6xd91nzQ1bk(Kp%vMdXBceiE-gc_m?p!Sf<^E`YcRX&@VXS%{c`j9^~J;)n%)TwaZRCucJ z4S@|jdJsxZ-C%M41&_jMp3|0M+R`;lca`q8U1wwCl~w8qy*B}Bsg>r|?BtYYi+0${F^ljzPRm}^ zK&#SbqQYzWnR<#K)s;gpYP&Wi3-)uls#h7R@USM8N^%E<_VVw!^9mgLQeSV<)40XN zf^%sE=6363YlN^gTp)f$Ga#Z!!#dM$uKzA_!9P1<$uu^6BM%=j#&NRY_CnQ=TaUbD z{9(>%W4==d{!F^XQXjjy+C+5gs33Q#_)P+PuwT#xC^<@NreJIAOU7!7;IvVri?<)I zE>-Q%`4jgx?ne!-rYh)WY6>NWM##i&tc0+z>U8O-xUm6CUUaYu6T_JLBvo;)rz8)m zhm;c}PgnF;dW<@*$qH{jVE}b*cq5N3zAukar^HpmW7G*MD0&|xu1@w8+OcJx)Cf(J zi*Rd~k(9ilhsiT?%QURMo2JmK)%MaO-`Vc9oS0auQp3sBOyvg~COf8I=QT8ls6$=B zmiRF70B1d-dx3JErpgKGn_oV#B|7YlE(m;@SUcwsJ{kK3sZ@x#X`o!??xvEV{s3yE zRH0$=KrLPIKVAR?B#CSb;NbbSlcY+8vSR({?>>6;h>F)FgjFpo0}ehwDB)5m0rwli z6hsP_fo1a)>V+bCTO7yHl9CSVHdh!@zbI_ z_NGDe4AdxhB6|p=O`DJpuck7YBEce%G*DM*)hZ1}a#g_;jOtl>!<71l80QN#Cp-b|b7b%yq0J@=OM} zV-kSSm;%Ta6QaX0m14_Bh#BdUuH#*(>d6uqM#ftc;d<*luQU^+CljWdY)#^Ak$_ zG+saJf55^7o(~w~BB}U{<9eTIYH24!amKZ_!s6oFc9_KUuhIPRusP1v{(Tmf!z?Hxw$Nqp8_IZt9eNIzYaKUF?y@5vdbZt;U?5 zVG!Tqq@8~5!%?9qXtW6()nFSBtBl~*!Vo{H;P3Gns%J4FEkLJZTvWq3b-JLIh#mSm zg8@!0ev~e?+eprI{oN<5AL`BtcZWd6RLP(#(8CN@-=ToCz~Qcp4mE6C>hFwijeYs# zIJLc{ZBIgz(m6`XWRyif?(9$%CUR*9`Z3L|n8?c#p2pp>7)$}9!ZEG>hfM{B*55w6 zyA?gGG#@UGO<eC7$qC^=FTV(&pOqpq))1bgr}TEdSO z&W#%1h3+03SYLPzRvGyP1m=cH)|8tXB%)8(Yez!Uo?cU!zH6N2DVpx?0Q(Fftq|;Y zorn2+bI-4QZG#3XK9eJPaUiTBtlU)fl^=-Rv9xN{=c;UU97lHfCxBx1YdpUAugW|$ez`RAccV1IYQAk@ui$9O( zyb+h-ZgqepIGu~jIkBas)O8l?yg8RB0v16mk?0bBlBS;%8ZtcFU2s^~Wxm&K)EdJ0 zYzNP&Q{}G1CVFqZ{y85L+76HFj=;3Pr+j=v0Xaubf%q^k-9JSA+ht ztjZ}Lg?GrGwi6G|5v2IgT}d7OC9A^^mTmE)qP!-(Ay3w)Fw-LrO2Y17s|2FI@!)#) zde^06PJpYwHXnep``K@`_NogO$8M-55ddgvSrU?d~Q=oRP*{`AvhWTcSwxa~b@`s0#Z zGL!X$`nR;1c}7t05_#xws@?l!CN$8Za=iR6JX$PyY7N{b+3lZa{V#64AQIN(ukUXg zfq~Yz+oH;}B?dF=9pfJMWH;1BTKyfkD-!9TPBaK9;~gxXfdLM}yQuTm zJOcR51S`4=2Odpx=(N^uYZaIy#q|CcL}d6Zw!Yn9QOzh;&DKb-a#-pNmcMoLYk|85 zqw#*F+tgB*N_rwHxoFn85>`iu^vpYLf}V61gemqfRpBQ&ba-rf&A?BTx~(LSEDu;f zV*AZ{*uU~-n`)*I-d(OGBQrO#(g_q~cUo$lp=z4RGh2xh1BmV=0oKP$BA0}$l`AX7 zjl!TuGa4(hH*pqW-a*QAkm8e;%O;&d_>ge02-@h}z~bwp6`fU8Kn`1<+)qk32P3Or zPKlUr7}oFb*;2cOI89#Dl*sJtQip}KNfdSVSecuB#PGIrG?!Z`!cUeG5%K9Slm9O0 zFgsGcgoa+e7c(IntFbQ3o%sCooo8d8lu*QS?OV5Ql{&2@`J87Hp4RIbeh!i4nd58mV7~6JTDyA4%Y+)3x4sm~;S4!=(%f^Dw5`F9-;%EjfYt zTNdf0bu4}d^ywy3;biS=@AjRtJ-paHdAR>XGJP-#C>!&a;2OV~d zs&}0!Nc&!6E$OGyUHzi9=aq2<}LxcPbM9G7& zE<=y#WLav_w%F0AKK(kW&~@!Hdu&2dTcojFBdaj1?Q`8lF*GzZJ-K#lZrnzN=JgH0 zk=C-dv~0}(+UUZf$HyWJcLA#d8jIMkET&pjzo7Q7xw;f<7Uq^4Ekrekb8@0foj1}V zNqHYY18RVr=r0hMYjy`#w@?^1$lVnd7E=5 zQZKE*v^NFMHgb;2_3T|&}? zl$LRf*VMDJ#4Q$MYrSUQIE!_KALqt6RtDxpJUOt+#y`8EnbP$FjgM zZbT5~o}6xIfgay}$;ohtO6y?=n1Rk@pg>S65A>;Fl`F(gAo~l{m0%+Hcmm&K>F9A` zu8QE|5}~KKunKvNOOR4ba_?nnVbbZg_sl313fMAfuG_1udX?p{nrA>Uv^sKv20lSg zk}Ne8%c7xVIuMeeB}StN2oZ4IAe|t!cbJ055ppDud7`f$ z{OBT_vDCm$zcFmnZoK<*1^Et`>1=4==SMV$T~CK8g%V^N+<>c4ct*b2A)GYiRWkYt{Gf_pjjmfW(8ykHW7nPRY zFt1O2egW);?x0$4UbjJ>ik{dC6I{llXl8cSu6N0~5Jn)!dOAEvAhd~Rovz$eDPfXg zBsaK~nPj4vXri)awX>NNeavbWTZbIDz;S)8Y37l3q1hmc^vRbl(>BIQx@MU{;r(*P z*`l`hc6$UnPxBiutBs=FZnaf5(&es>8ozl4I^G+B0^E%ocjCq0-bkt|);H)@dQfkg z>AW$an_cHymNs&XGX?%hxL9pSkOBNb=JrOE;=-UE&&)`*W{zc$J#jCfrjQs5^H>_5 z)t;T#B}2v1wGFe#q=!43&!$Z2CwDQO)0|JXT?~N@igJ+(xfWzd9eBwn;pc8nQM>Dt zRHntrwR2wV8%TB7-N z4gafbhfq2ed)Y}W)egLPk*4q|4y?`Tf789~?uPcWP#upV=ExkIY73U`j3XNv>uI8?iiuzt$N1B3RJ#8py4b-1 z5Nd`N-WXWJ0tnN?gE8Ja*7#jGyt0E1Gbv7ERKfyT_1Xn9yDaZN$LDjsj5E$Nj4b+A zX1f*OBalkJbONz(PJPw;Xo$RyiPBSV3Lag$S!%1O%UACC_F z%w)z{!U=u#k0T3uIdGiddD3rrg7&5|@bnSsH-C3J#%^Nyk^hBTGMSo14=&2fqwDFi z0CjA5k`Nac*IE%Lv!h-Edu;)}r!xk#8w;n+)NyG9;rS7|a8#^q4&@en3L07{DSvPoa@n)2CP8cTj8>OHfhQ)UZV$4If2D2iPx2}9i0!cJ@z>!pqB>+5r0np02K)j`+q z8ZpaV(36x0C1+9zOwLWr6q`FsK0>FBPin{<`*2-1|5~k+|AQQj)lxK4!&S9cr~#U? zX0Bt9{lYLy)6q{iS^Ij9U76n+E3j)hpQ)2qUoK@@H&hWb>awMS+rVhodY@Fu)O4rX z9X;h{=i>eewjj`79t7Z=WssQT%$YMh=JVZB}z8?$ppEE8=Tb#x!uk)GBUpN_^a=BE#MP5uL9fa@DpF`W z0VKg$c%uhah0ivcUeqCH10NrSZ<=4o;IhF{&Wox zEaMzw(QH@qa+yFDIyj-2p8~`els_Gx9JX~bdwWAoC0kR>JwQ0km(st=UGGg(xtTiW z7Pg-AAsmcwpWFsVJKAN+ z4SH}s2aOSW#djXZcjbH!hW^bZ2)cWs$;;$CO~P~LIpe;sQOHDzPSMj-Nmt?JbnYlM zIEG8p5T$#&^Lh6Y%cL^%wc=cuwy7z;zJsX!@@8kJxi-GhK*j>|?)qI{jPvB?Vz(Nc zk4i1SzZ?|a@hhzy9uOLP0`Vgw+MkL~5lR9OA*nKyl{JTW)}*sMg0X3=st`m66%ePq z0(i23N)!>FRvD^-E6P>PQm2RM=ueHJZ?REVO;?qrla0Ott>yOCw#J-Gx~0id)YQgW zBc6>ib8&SJn>N+gi~E~z_3u0yxT&&|ngG&jYSQsQ_Of2~N0C+CJPRG%?IPic)851S zvxqJ<8WjUAXkgpqCVpJ6e#hmUA#PV-vJ&F-z`sS!Z*`QGN3za`QVrlfyQacPCMvGH zTSFF%Xhc6uiiVJnjO^sCDpCo5n%MU%D<)&H9*{m&gjboc>_M)am>VD@jf#W4;hJqN zDH7}}`qatfu3j9--p4y9*cvbNpUx5p*h#gj^_uG{2}TwfNS`4B(XVOCcF~QpV#p3V zFwJtaKC+D@P@1rb&pyK>g#;wdn2Dpj=3Vz0B>cM=5~b5EztdTQ%+zFRu&pGi;j z6r<0gyM9{H%|IY~F?za*nc7(AX-YKayBN~=8GzTw#Db0i@c~!mdtZ9`KC!T9&FNyi zCfN@7{3~KUGZ0fswI&#iWenKxEH4yIPA+6Jc$2r3nQ5H3cuU)cl06Ur&_0w7xK@q4 zNxYUAxQkUG3d{cyt}-RonQNWNu-`m~uTL;7GI!qdKL97kV8|mPL$>e6JDxS!OMbND zN8(ue=vnzjIJhuLg(yJIvtT|sruCR-zR8HVO=VO%DfFs_q4W`OQUtL`b&EE8ZfMBH zPP%?BVC=eP1_b;ALj#ClVKSWX#$EH1qI}Ua7x`zMsWwKL8b!Jb(HPyzN|lM3nQH+z z4c6yv+GjC(ZGlid?M|F!UgH%sZdNXUuYyf17;0|`8yKB~q?Msb!>w7o(mk1K@^F5V zygB7>+L7~JSXg30NO}yC7Moo=>^qMKd3c}pE5KV)TE>`(H+og9yChG%ggJ{(8`|Hb9^92E}Wp090zPnV&ttowhG~=fs6+D zEt$(j2J1TO=6z_xnB>NHvQ1sOXsqp`?GrCS;5c$W;k%SCy~v$;?j9F7C6JHWR=w%P!J)J1QCDG4V%i6*&966mC+e;X;zX z!g_m4M1R=dfF|SMDo9&v;WYh`(B!0egyha<&QyE%JT|dT;`z(Qo`Z)-0{0Xv8wLjk zR-%uoH}Wu!_V)J9Wxa>Q#&na@`1p7M1_UtTkRiIuW+A$$wD8FDBmTUYXhTY`U!-iu zZ~inQ+DL*3dM@BnG?ZSPpqj7)PfVlDmsApxZ_z+Po$V`tyf%eJ`_7@uHhQ~b;ic|- zrBcGHlDcYjCPIXMedjD4L6xle^sJH6Njp0dl7Z*#-8&LUhq=WF(DG>A=0BBZkt%%q zInimUY1#`YO&t8QR!T+k-0aKr>|WmO3&+XH+nbz!n^H&`#m*o0$SD8a8i7PeUELKi zcEPWl|3Jbi{)HB1{L9*S==m>@n(bd`p*p=ww?7m|PZ|i5_=a%k2(&Z|mpgGEG5PH+ zQpkybLV!BoX#?TsG)ka|muLO{jXRaK3Xgw(?Ps!op~0&EvW@nm67_5@6ibL9=LFhYue-D8uAnaYBWHkn@{>HG7n1%om5zdGR5fJenKo&+C+ncSI z4f@qww>&D;-(R*+tpn0a!OZesH?4#6vb0O$Kr2gy2+>K0;`aq7Fi4BOF@lC7P{bYq zHJ4=2GXt8~6*#R;nPq=KcgBg?WSBjvf<*o7bPT2!V)G412Tmc+4*3;$(%0qqXt|RT z!X^a*M3+!6Fmm?*2Z*#$0YxMOmLn6uIZQPa0jv?s3Tc>qA~pfSN2JkT?x#;d6%`eu zj^ihsTwJ8F&g(NsC%dPh<^hs47U_<4*Mrm;c%aw>=sY)ruxQT#%L9$*_8?BM!zKu) zITN^l#9IOK50Xll%`6(lmX+;P6rIYQ*xg>!EU`{Rh=3qen}U9CGE!Oq2mhe*EUMo( zp_p?U!D&7As=Wu)2wOla_%`!{a)@Bl0G;{(LPDhL0gUwqbRS8BR0yI)AtER^4O)78 zRr|7Zr0sSgnA?EXAT7{X2HEN$rFZVk83cKU_>W*yMEnp;Xb6jXKsCAG^FbOh`NM-o zN=P9q5(Xjw;*!vF-xdSnVhFt0x@vz|07V)!$Ls6r42nNX!q6ZQxo@*Z{?i~x%(F2w zGo!JvXhK+;1ae*qdA_~pA!4z0DrfA)+!mua-F|TbJ*msS~{U8KEa`kR%;rn}EjG~>y zq1jqD!e#+a4InS@z~L-P(Jgjp2%zhOy>fDPHWio!O`{^)2FQiLVOk16gh=KV0eRBi ziq?{$|I--^3&T@dTFNRQATZJZqpH6mOHvP)-R%`e=ao%@dHRQ>F_2Hn4t=>L9(EN-NJFk_TGaAm|XnDus{hd;mSt zxV!=Zk_1T0P<*Ne!^>kgr;K1Xw`@7olax~x5%RR9rDbXC$2YLGM?2542*V|_lm7nF zKyr1O<5FG7>Ht_XGc!bbihu)ghmu_=)Py-J`ZpZRsd|OMqkNo zgh>ugl0aBel6GJRNd@l3@rA>`o`2&Jc3LTRGz?e=(p?B?FbNGtVET2zG($Q!y>sz} zL*6X>TMs}Lq0WmD4yu4ZdFM?JqM{DaYkC0^%L%$QQosWc!t>7;2J?)M>?zk zNX{7shzZ#;y8yF;fK>+i`x`+@0as63v(lecKjY<}?RI{d=)!s#=#Ia78162MdcD5V zka#@MWn(UMQ3R5IJY%CkC@F7IYg}D};L8`)!jjRgsb(1rhTWBJ!bNoLUUgW-<5JrIWevwi5F zB<0j84GG^MVONBglAR9OU1(B%(31JCFej5uwzCM-T-{pAuM#JUb|N6gd zXaD)u|LjVQB5OJcK*9WwCGi3nN0`0_V>s(%?EU_rKyG$2Ql`H?tgBNII_+l}c>X+N z!Q=ll+WVjP&?hT29(V3S9zrBL{m&;YojU&Gg<4}Z{eS+S8~---xmVJyZ@Fq3u{1u* zP_b^l8D;bR3CoY}Ff7=CB7>XBRDvBbbX1gHq%7AuxgCxGPVm|ISi;SU3@@+q6ZUA6nCQv06{u{EHOIokb-(#{Ic^ z7aPBsOn~&?@w$f}m~YWA{`1$r!C$WqUwpJN__?ra(@tE{G1(@l%d4(FXIyb~mX2+X`Md(?cgc0QNMt8JnQe#oe(VH3%Q-{y6Xp^5CJ=j+SfN`m$N3e4`ph|d3$`V zJLU>(08bj4)l#zGe_>mWsO8F-*ejf%XUIHOWfOFIVmRmV5#~WK_QS7l2g0g6?uMVK z*M8w`FGW8l$&sa)zUZWrzRO%zd-n))>i9=LZ{y$V$c6?>h<#%|28vS|Lgu#uE6x=*vrUml03&XRnU5b z6`yAF$wOx&^~${b9L4xIYL>c|(fOKUsr zSclLLsd{&9wqNEOh)wO_EJXjjJP!?4r)fMKl*&@`U3|Ib|4(CA9#7@gwmYZL(QYnN zp+cE56s2U&&XA!kLuR4O%Ct=(I=nJ(vrJ{m6v{kP#>|9lLn3TWM8-_tz1q&#dB5NL z`@NsP414eAv7Ys;XWiFz-Pa|Q0@t7SO~CMzz=K~DhJK#&E|1^OxG&M=KT$DA$9m*H z(plha2%&;G-sBe^ws6V2vox%cSW0v3V2TE5ZCrvhA~WV7Yi$_CPkuL(Pzf!@O2>4= z=v{Zi!fP{1n3kqb+|Mni>DD-|(69(xV-7E`DoeY4QG6lGBFo~W{nT(mgIB8CiF9X= zPM*=3ZqeD*;PH{$3kMZ-PnUZ52L0PKJc{YH4qX$Ke$!vZ`^l8_EU#;<2BuEE@@MNb zS$^8?Sr<{cf!ymh55wcXo|9$_d)1IS)IPAM;Bsx3bwlci72z0uC@Sq+hhn;M(eEW^ z^1ZyIbk)f_@lzvBarzws8E6 zMeZ#gi{%&i?hxw#U@6VCi))UyKi-Qx05WRdXqd0inqpWEIBHI>zPjDgx;G2wt<_c+ zkl@c9ALB58#Qf-BC!MJ4{j#BK{QVZ`v7KO#V~!N&^C42g0c*PzzW)(Z80}qiRfzuk zs0ahQPNhD#+>uJfr&S@PvBfj#w7hwjQrAwdSSSqljxw?s*>jv;>$%{qWvxM2?ibNY zNj$K~Kz^38f1-w){+oT*bDeZ4363RCdQmrQKYmk;W!$Vojgu7a=H!F_V)5wvMP5A? zWx&7^x%-t$vb=_l=FXCpA+b6E(%9KB zo25}o%YX;wvqFY~L%$`^R}d!x%hn(ErkeaglCxXbR1@*fwO_w#Sa%PvlyrVmNzMl^ zH>s2&D*7dlJZ9{{Hfas|J*P>{fBG|TuO!dgrk%*czMLW%e#{*?x}NgF3VTq9wr~9F zhnW+$lhNKCG4C`?tH0dVcq-4|@WQmMF7ZIum;~-?DmJGYr(1t5uzk5=JvLzH!ynjK z2i7n=ruT6nEwQ6#L`#UKLy;@RXed=m%wt?a5TU3}Cvw9p;7|v#Powyqz+|GlZCDCg4NN}9tE-7Cua-gIMa6azY%n5w3p+j4N&f|6Snicofh3`m3oC<lx8{D>DQSDqE9;9IXd|n z=YwZu7Bi%?_fzK+vTSFRh1mcJDarDYBA*U!Wz^1H5fC>kz(4plpbDUB>ypKcR%t8afESHBUc&O=A^ zE2ZBTu)6w1SI5@l-xq`meZJ}7`OuK{{giA;lUiPd^+91Lh2pQ7JkcHTFQ^=SyF_Fy zx^eI3Z>PxpySD{sC$H+5;NP^Yuf(0SeWG7;6Y|$AR=-5I;M}9oY?SIuO;d5I-{yKAI%h{}Wij z!CnT6L{%KFOC&NfA;rSBTM314Om&=AO~*-79c1bO5)Ux|poo>18`^Ps&9z$osHzNd z8{|`jB!^IWbb?#od1zt)%Zlg~SvfhVLlwM@F_1_OuwBXIRaaoYQ0yX{5BPy6LnP?X zk}E}AdaD98El(C;Am$Jh4kFyx<=LTUX4_$34Sid-s>lPN{J@x~d>7C=$i9H=7J2Bd z2mu>%6b%8CEeh-gybnGxoK+S;3{|K{K_!q4xql>R4Im`{arCSSBy8 zUC)ylc^R$tI!AQIq#mQ8NrfJtH0lVHQG1vL%MRv@;5XU_=);Gs zI~VseUDH4TNg5g&85fW#C29zQO8ctw;!C9FfS^}XP~T|2Hu6Odhlr&z?^8tNHV~Go z&m}34qvG@k1hH)_$lKo0bKfjLNv(k5AXXbb8F{y&f>jn#7AT1pZ1I=A64k-O5h1HK zFHB#nUcdeXGND>mq@<)S4z9E<1H>6v31RkVN}-5HaM7>)G0la>6}Pb6=XDD53w~7z zMp}F()ak(0g+Zc(x~`Knh(`JVwqT|IPUY60{v<@E67eS zr6DCLiQa<@C^Pdz{)dO=&B z@3LZrq&$=D3F;8alJB zmH;|_Nyrb20vIVWwQmH>D3Om82>5gf5N!c1Xr(Z_b+119kq>n2%hgp!bdSKbieN={ zstN8}B~OoWydbd2$Go5JO!5IgqBImmga9Wftie6Cf6%3C?RxVvJ*f9lb_--41F9(x zf;}8i>jeNZl)pN=v#>t?KM*|tcw#rWY#=$KV)e5hG8+X)RK342~k zxKB+ghX9YVTS1nhYs+Ba@kOVywnCL`A`XShj0ok$u5u?^P)5hlvh9Gj!L|zud?qBb zQu48P&FU4NhbGORUx}ao->dp_*EgK}!78vm{HmM+>uU5J9UT>W#~x#eU(VmS(eMv| z;TGT2++0HUlB0g71Nmr;K~aAv-Tha5`oBOLAGKbqpdX;QX$IEMV+jx&;31$dEK<7XxS502B)M*ey_@1>ltwzacs`Bh2&M+Cy= zKKuN5pT9dZ2bVp4JXczI=Gs`w#fAXlTwCM&m~meIPFz1VNjuq2R1UWG<(V=H1M-T0 zelPud5U?%$dr8DiY_fj`9XP)RqlUyX!O+2)34tB) z)7zxH@kebpWL*l!75;`zSf=<*T5mr8w8lPG=@r^z8P<96!3v|qr!V_T^mZ~4#_60f zns0)w32>V_zj9xAV8A;v{`G(&F2F-Odk2ZOp%J0X{_s4mYgDIcC9^NGT;<8*{R0du z?FnJ78?2E>O7wPcrg1t?3G)Jjhtuo(2U3R`h?ng(fJ1>V4=PG2-|SnfuxU0px^iK0 zFL|Gkdtr}P?*umMSmBwa!kEWSDnFaOckLRd_dcTaEcXj;aH)sry0MVTQ^E=z2EP{L zLl{jzC}u{hQW#I*tcyGi-a$^*uf@(s|C>aSSz!Ks^ZK9!F*XIa)?&sU-399y;U)5Q z7LRY%v&CSa$d336PSljqUcMVJ#!~ybNSESc)9#45nLAe|r^)w*hb!CmBAx%{tYFhD^)PV~Vui0n}~8H-yv#Qi1C<=1hTYxJ9yl`qs* zgi^gOWUp`9UH|ax6EAN{%TR0P;czIRM|y(Dt7f&?_@?q_VYm=x^K!lF%}eta1LaMZ ziZEQ4e0>!eW96{_iy4JMS}FGVr;6o;OnMoisF0)U-ByN*5hT_pZ}XOe!t495xC+`JnHwyI)5?#bqA7=ufWS zsx!W9DSwWdT9m38%>AJ?w$Xd`?7)<7^pToVKM&)f`hHWtBjx%4BnHU)sr!G|Ix#^@ zzO2=Z3-{RbOogmtz#>s5{B<$IL+1l?VQmR=6<^(G1UaUn^vLVcE_`EI-Hon&RP70b zjc3Dp9(WctPEAS9>3f(+5n5A=qKQ*2&6BkL5Y7DSj&q{L$dP=!R?1`=pU2@=fAsi8 z6}!j@?uK1IL1om1t0=K~dc*Uu!9neHSmY+`cPOQWxbpp!DgJLI(rO7xG~UM({KDK~ zA};L-^$#|Gd&G48D(RyeLH#(XWqv)~z9wXN!;@5Ct~Ssf{^y?Q36u=e}bp*8RR*8k6J3BV5v>cjPxfSP^|yV7DOo*$twf6#y5N1MnF%) zJDteoO*Yl^P?4p!oHpsy0~@UASn=ugl>s+mpX~mD3GE^B^bSs1PztJXIbO-LRv46U zSWRbJiF4xKAP*^g>!Agb(f zyNr)rY}G)e6eiH`g7C99U(r_*B?k&c>m_WsTy)?D_P$`}xX^oB2D*!qi!w zVqvRos{V#N2-_Q-32^#W8zCPbp5l;mf{RqIEwG$mkSelA~o7mh|$?`a*OnhXvkh>dDs`~ zbpe*|gnW_ZEhA$G#_9W{lzr*$R>`XB=9=vA_{{DEgX;&kAkF%4;Y%fb%aag14 zftvO6I~VgHE3y0)c?@@X4F06}(N7w`g0BA?RovllsAU{ z1qeV;iWcyysIL#z2%xT12Js{?v+>YZg`z1SB|sO*b`*MnqIE$VActZQAW^pf8gH%$ z3!6i6;{4h7+(K^31!z?N$KKC|n=Z^>*K2fi6kJt%D<7@@tg)fF|0$%bIXV_Y(_Q%N z!!*UXuGRT5LAxns#0`V*oH85>g%_jjIqQn02x!u?8r}wzfL>_XM{orAV7!E}MlHRN zW|sk-xF$ezLnBHD1O%e|LSQV-pgIN-L|apuyIv`Njt4}uvN&+mSQ_HA8IzrUYz2 zeaaF*e}%3lkf^3$->g|`XS#e@{1R%2`StL?h8Um)m)kgOw0`;!NaULjj^nf_%20M5-2y+gc{uLC_+fttP*KyFNTJGWq%QneOiH z=x5K+l2Tt^-!(L}Ibv1p>+I}|;|#ZPc8(zui6@zvw`XEt4}bIKO(Q)1GPHL=Kd1~i z-@MAn>HhE`d~$M9TSuoN;$@S(lScY&5YVEeyx7=S==0(JU{(-KMAz)o33D_4sB+i) zxfFw(aEk9<)!om_A8+$WPqot?k_S3q%TJCz^@(vnb$V$F#P#VlbFwT)L65F&ivw(FqCG^@V78 zhnDyiv5i$N!p4vHBkq#(KG>uXo05%I$xXd-W1f2}?UnZ~xr=szfMcWXeqm9OUe0(r zGalX;B70rpc6)pKcL-js;yDGssTS^CyiAF9a~{h{Kk-+OgEIU-%P!_%awfazsXFw^ zU9RF!f0hWV9+l?yyG#l}o}+dH{lP@|sl;#vXi)zC(VB_O!;PWbm((a}giW5jz6isHo`awO@q?qm$buE3J6_m88Ds{{U;Q4H5tV literal 0 HcmV?d00001 diff --git a/docs/ae/udp-src-trpt.png b/docs/ae/udp-src-trpt.png new file mode 100644 index 0000000000000000000000000000000000000000..2d8873fb6f5e5d3f341c8df3eee79bbcdf2bcce6 GIT binary patch literal 55991 zcmbq*1yq!4wD#DDf*5p(gf!9}0@B@$f^>IDn4o|(NVmYyT>^q20@B^05NG`nh|}R1=iwEp zn%bA}-&rS7X*CRZJi>Sw0DluY-_vqdwKH{gd*En-FtfF@F=28tb~G`ubuzbeUO&?y zgh1RxNQ>W9b5B^AbbF+xet7g_=kr6;J`1{MQW&S`WU~8SVy4o{YK!ZiD?U}AMa!k3 ztfuxWb;SM@uDwfsz#KzSn%dpRJ&ZR~Zb~UWrOK$3_VK+*Y0USKt?l#jZ};!_*Qyt$ z_Nv`}J@pXo;vEi+ic)7xiT=60U7&JA%I`{><`EfzJOx8VygzWSu0f=f{M^{s>f&EF z-`idd&nVEZMOBR3Mzk0?IGCV6g&5G6(8Ncm4f&J%WZR5aeZ`Vd6}Xtb(`U$qM{sjQv!sS&zn4axR{fb6h3OL5ke-I;Bycne5h4l zUr)sLFrdHS$B1#rQj*tNFLjLG*D|Z)3nCI!&E7`A!pQCItPkf`PSHPYyeW-|fzbma z%{V&TX<#nEK6maMpXX+FZpX=2)Mq0#M_IH>#FvNiBL*#_T8w%RcBTtWJL8smQpI+= zeGk*kk~}_6?Ytni?U|@{!o|a*k>1_xS2Vdp;5_Lv%$vZ@%&bscTufa6Z@oP{*c-2O za&$z`fmFatZgIT4;)C~9e)TGxD`gH2U!Mm!G>4~wQ}7cnnh?Cn<+&OdVj z+kAUv)N;DM?qdjfA$pg7&&Q9|x)sXNL_B)*h{L4)#<>d@H2W)Eeqdr^ZqCIR(hCV0 zXzIH}fuSs;!jBhDgj(H;XY)rZ`7+F9_{q6T4>^cjW>Scg)u&s<#(5_BXnhpW_>35A4iXS65f>P9Gg+ZEiZ3oA-(m zb6JOWnlfzEo<3fUa@ouesbKd_6Eb>YQZ-&?@hbMM%3MP zO2m%_2Glq$2R_wL=BQ@rY`iE~S+Qm^tk;9-6LDF;A>y>85*8NrhV`eXr&j?lqpyGK zcm=XW%9*7Sqod!#^nd^Q#Zx9JDQUGjRvw;VJ6W@|x89%$2dvy`m>KOQc7JxTL?tHX zU%#!8?Xon@&({~*;nKNL&bEjzHz!z z$$MM-0Jam(Sd8cPN={cIzvfb0S7=yRDlUV@C(}}kU*Bhth7I*pIhutAY(&9mQ^mY= zNnge4-jTxb1m7nkU36kwjEY(Cyz85qL~(E2kP0uT`id#IGvP!e;9hk4^y%5TxlAx- z%huPVuR=mJhYR#&1>NlJi>_U}CNsG+YL>iIvs?h~xHv?MetCC}mEdyCdi`D`ti*c5 zk-;TAMy03cu{PhIxuWK7Z34$OYB#9;8V*j%XsHy#FSj!?sJy%$KFru&o77F~21Aqy8lvm$O_z*HNMJNHG~8P1m+ThU2!hjZX;s!QkKMgoOr`0BGUCbRH1_aL*0`v`l9oFyD0m+o>~QK)pIBa=rEk;$oK3gRcy2}5 z`0xx^i{<6QhN1V5&#ZcgoFhBty`k9G0f$a zU|5S$ng+d#z3J4_F?xVx2kE_%x0Z)O1>WW+7h^`nyjLtvI#+XWIB3&eZ3I2 z%TF)_^KDrCg26^4rN;H~zfgexy3Xo@kGW$FCXxt>mM9 zNjR*@K@#cU-3uBHJ>eFUQ)Un+;`rSP&=CTB>OFDsk!dxJFbK}nG&Gh!XWQ6f<>(n1 z2dW&tt+;HX24zD8eg@PZ9eSaSJu``to?fQTbF1&!#p|=*zNLTpav$9G$WEorxCTU* zf##RjP0DplIEc2ult-RMQ;5AV?@Hi(_4@VD(sV_Ud28}EeQ8JO%x|@UXkEAd;IK6BL*Nr z^h4l{rlln^*9y@NmZC9d{PX9}1f`3|Z>_a42nb9Jaw1H??IC}%YUB=Hj5>bjOVT?% z{o`lv1^4&LtJ^y}2gR-ktC%L^cEnixE-~>mp1I{Uo>)2gHX}oPuL97Fy;K073o4I7 z%pYkXF8*?Z3YL6jwKB@QQYYW?8d*W^-uB@48Q++gQN5foCnu+^mW5=iez$4VuOHvQ zHDO}eii-2NGULyiCVg0zeMqhe5}gS zp8M~?>Y6h3T$!C6wLgm*6K}s>SF?+%56=*dmw$Qj;#if#Q?fS#(#Ea9!<V@r=|bCWWO=W0UmMuD{FAcX05u9Xg7re&R>V#3X;@Is4$;`SYk~T|3o>8&a`B zG^B!_;^ZvaX)abni@$kUkjhF*<&$O<_MVN z5(MtCY8qHfE6B?B+g-q3taCfcm5;CQU%{V`#27EA*|nT>TKC?z(NzvpND@f38_FLR zyCT<1d9({50F|$iuVb}jmdsBQ9~zpnw;r3qjnAa<^aU2_Hfw2y;~$$~HY@k+=$@Xg z*Do{>EVrE00jnRn-cP0~WuDFFwxsfv2RxqtxBdMCH270URFyMf z)v%FVdE*OtO6-wcg;-B7Ev-}{pIh&WHv87tgyz;jkQs7DVflRNsNO9#_Bc7V*dW~3 zX>9RzB)h+U{i0=LluP7y8xp*}Hq=V)oiV;v^dN)sG zGy1N&sou-yP}|$cNG8aScj2S&Lzw9nxok-#szv69TM>$8uB^?mC~ zO4znG)+QB@mxD--q0fNjFg97<&f(tc?fw1Iv3ru5yH(%6+vjVSBb$5<~=9Xnd~nOj((OVei8=l^9##A6;R*3d0AewkdEPDP4ouj!)g2y8j6`5$o`De zB5fctlAhFIkcR78yrZFBsuXNBKfApTFw@^ z8oP5b-s+hCub6n2mcs>!U}5@34RF|}eTuPd7#jAIr+k)~CS6-e zJ-?)uHugd;lYQ^f^gv<5O8-Fwjb=?(-@EVo)|}QON+AkfdZ!-yPJQrNKh(6+C>)x+ zPLu8J|GK@az3*c%F~`9Vi@4oIW7JSxZQZ)8(>yINuk`h`YR%4Dlu?emWkP$K*X3h5 zlnXz5%imqdA5>FSNR%n~IVKO0QAS!iXTWXAvhAgGdsmmMd9EeFJm%aBF@A<>w(8p` zm3F=1jUOxRn!Ounx9Yb6!zq%Xl*GJ*S@z(-(`9iawW+@N-~uf>r7Fwkr2On`Jk+qT z#$@ddEqgPM!+zD&qXQQ~UgH;ht}74i)~{brm(?xr^S-{O5yMb})Lak0t7^h&OY-*O z&eU?x_8z2HztTfC+b4!^45mHLD79kN=l7+dA}Gw}j`+RrQy^S)>f+YN-FNww@@u+oJL^Ou4{$cav^))Z3LW zt@mL#6`W3bPNs74>Bz`_dS>S6#`~dE)YLSnVM8C~WQM}B%4(hGl>Wc28mgE)B~Q{{ z!stRD9kr;o*-VH}n0t_skqSDB+c6pB<*hJIDm5AnEL9e~R)$0|1O>0UG71>vJ!dG5))^H@YY)qU;K^74bXWus{s7-U;R$omJq_bj@* zYqvV;z3FLbX?u@iy!V_pKV2ak9PYcQZ)faIBvhxK$BPTzVPbsR7)>z+J$ z^m})Rp;u_>aPMJHvT#9ZY3c0hw{$?6mOsy2$(|uEKDTL~T*YXgyl&wmx&CBNotc@L@Qi66{uFFd{%CD0e{9mNt@bE< zmj)OM^%(Y%j}|~VmU3UX&em6kLMgu5p`OQQ_Bef~8n$-#e^c+`;f_W|L{R29c4s2z zkM^cLH@3J}#;dD&0rC>@S--I*CCuS^wwuYz=^<-o*YQ&9LQ@2inTPkF(WfIqRI+(1S~CWq2A zDUMaF@viTALcH92hli)AeXJEdzTCyVfyY-in5)TV&6lOD6#62BAhi)-&vu@+r|rkf zon8knXi6g^i=im;`+S1mdjud(9yw|xwzhU`6p>&=RBR3;6`hYz7SCV2h#s#*c4xBL z)*WDPp$0qZ(Rr0W`6H3UT0Q0Yn!A%;n?1j34=tv-2!es#uqP;AX`>u|Th{8gZgh8_)?FAAKL?F%C2H^%eGC=EB76ljjcUd*X34^n@Efn&_8S{*<~w zcFkKLCiMHtEHdk#wzI{_Sq|2?lqar>Ud3))RWl_+V>v$*{#aFUh4ob0#KZ*K)3Aev zHBBw8Hy@711J(1Inx3#7u{0QsvYk6Wq}n0NQohsHeYCM2_!}z}hrE!tyIW%qdv)%c zs;Bwf^1^NKBmleT=H_xd)FNFEHX~kKCe^S!BKQ4ez@$@AG(5Munjf?oq_X_EyF2Dp zV0UfdBWW|=VS-=@|N0nlj=}W?a{3K{Bi(vEh<1#2FN8YRrr_#nNHKn8igbrq} z2a|Hw4+vQe=S5dmR*sd~Fysy*8=oG%eYrE8C3fKoa@8b;cT+GbArH+^v;%ZseidY0 z@T~bPq;D~A@G6#S?WmtNP7NCsOY!a1&KK@*+JCtzpXAAVYc^RYaTC+6ET^Rq1H1 zI5`Z*YpqsMQ8BsT)R_zP`{A-HdvX#Hb=}X4gxLc#C48>z?naT_BD#FOJn;3<+-llj z$^$W6yc>&CZ8IJT=n+hdbfm%1uD)Jm!@x^^M?QgyoSa-*MZDCs2Dq86 zfr$RCEloo@LBWHSCm$SiIksl}(X7SR;_rnxejAorIdFHwfC^o{S58MJB^4Yr93qdn zH+T~vMd~Y;C^t`CwyRQRPkN-H`|T^5E{Kf0k-z`bSLcpz(pvSp(4Kkv-C=`VH2Ck$ z2{A`UlJ<5L35go6M;pOpUYmw80gBmE7ql+|VW!e;P{*-8T{!rwJXco2v*tC4kY3v4 zLpF<%X-V_sGDf?+AI1)23`Lz}`6k z^t^Z5R0M&Hl~GiTX=rHBDYwm+e0!MtIdq}Cx)25bWcc{6QSnfTC-S)r$pTcBjpK5$ z?CeZW+x*ycPiMOY-}9)>4cGT)hNF{{dc)l__bTlQd0Fc8j96D@AlVInE5EgLTef_$ zv(#?hlIUt`yhQMj-{Ui+yI=-_zc(*48B{5;C;7-MnRU@J=Dj?RO_YI7pq{8q(P}_` z(V#C+8HY;0BHRP(Yr+*jxLwk*IS@o>IlkSkEO-9t-H`odW$!x2OrfI{)lTm@6G(>v z=u2D?W>a5HOK17|QoqW6W~e~-yfc7|0FeBoGz)Zr>B>u znEZxeFwLYxC4@Uxj>x@}lh1{|x)~!4ti!v;IR~hxEIp1opPq2HTJFX`Ou2CGOU&cJZ@RbpeP!^*%csM|N1V;V$uJZtx~-K}M-p7N*E(m*Tg?9USe zV!-wuDlo_d7K77#q-eltWu#udft*h*O%U=mO1pQ(yMYC_nkcSt{H{E^w?Ot-2&LSu zc|d%Qw%J-GI$x`hL@?8$^J)2aapVjCvo%FfTY`t>6`;NJ$BMqah%-24r8!<_g(lxwgNiWWw zv(&Qp3&*sUyM4@bd|^4{vs_GgH=&oWU#GJ?To6^5oDBFpDn4Fyt@`wtW#05=diI(c zKU4)5r>uX;hQ7W;{!mx9j^&>JOEV7J>5C?#uW*eDCM=hahw1(b(!HhkhX-@pbU^5u zN=Q5hJcRZwGV=2I1wWbJ=mwMM#K&c7h9pKO_hv7oiOS^VJ}ot^;HmwjuJTyEmffkG zi&|CKY)(4786LNR&KCWnKQD5eDa&jzVvJEU) z!`IDU$fvKo=>N5Ac5%=+i#idD=Pjwst(!;@FshOBKCf)PWKG6ad(JIdP~#_&l2y_Q z$@>_X?F@6`U7F@ak3Yo|U_@}$p{MtD+wJy_&irU632_e9hQl*J#g4guv}sB;_qDsH z)5({hdGn=THX}JrQW#YW*QVPWKD^cS{ObJp#p(~z+sfjFWc>MHoV|xr%sE@4gqPoG zW1(-LbkNikXFj{dz1Xi9P0zxud3bn;;=b?qdf!J2PMxxk&r-+4K%t=_DZh(`g6-=H z+tFJ~)^|2{!YK90E=${%&4(tprD!t|GxS+jR3=^u(0aP42!rT_-H7}p8Q>};9C4Rj z_ZgqS2dDI6^c@j?13Zj!l!Xmh~rT+M)|uk_56!3sJJoxsZOV%*m6yb&v_-1sz<8b@4s--g1O+>cleE^J7~Qr zs69OYdv9-ioDnbZ!!~atMI-h&la2xMf$xzQ`Wm+t8Hg*Ev)L65E#z2piC}Q)O{j56 zEGFTo9OdBU+A8|vFfA87;`7>j$#`s{;oAx%003=Pt<7A-#MODXCdF^}m#u8(7 z7luv_T?);leZM^VpxKy~RtHU$`|HXxE$vN9smgqYSpHg^vojawhr&Ce#>hQ8V9A3|r`)Ojy@uH_m z@sfmuX?K^mj>qEX_5BT}{{#gAUu0sT!=A$8&b!lKtFHv?9x7%GT3-opHJgv-;c9kt z;7ze3eM5P|MIyT>r!?l78+*#cq}%D~OQyRSBo&vwlT&(1a26C+ZCdBccJ{%0aRX&x zM#R5HL)MkMB`423D&zdN($sfA^ow^Th9oOY+7Tnx{R8pwK-SuFW_n7tufiH%%#^Qo z^uI3^j@Ob!pPMZOpSN)Amk7_87c%u4DdlG$o~mc<mvGI9dKV5BlRSEGxOz?|0Jp zZSuW*5klSMtP(8Tfm6tC{oH1rJ|mt4?{Oxoq-}d0d$sGc%Q%c}U+z2% zk&j>f^YDEoSBGL)Vz9Sd7*OyPJNw>Y*u+suYH5|ShMjxKrEdwe$d8xKPbQb&={bE5 zm<7!NYkvFWlhmKhT$`1?2F|z2$GU14#a|sb@e5mJzD+&8r(aF0jRdx|iuD;`^Z>Va zwgcdKJtHQ0sO%P>6j0?mBk_`pD+)@7#`T$uL?=AlI z@=aLoRh;i?LnX7${@-rp&s%CYoKV{X2DY5}pCiz56r8=wuaJAy!*5M3+)Qm>e#z$0 z?RP5sF8=C6?s9kj_zB%TK|B*0|DBzX1H1%8>ogZ%IR;p+r6}VpbOeJDDhIU4rbd?< zdKmenhX1^T>qa99#|?YIa=~n{>=RB%T+#*focywGkvLNN&^)E#w#k5%XA($CTls1v zt)iiNh`rv7u+$)>K zteM6w0;c0J`uS6kII6;x(e?~Zhau!m%l3l1T!bw;dg=ZB#B0RzyB}g1zt{)u~M@TsBPzb`Xq&}4_~K|Q-Wdw zl?9XnJ%7)|BzpYjTZQ7Es*;k@OJTsm^u0S!okG`ux7VlFpU50fLPURlz}|dwvA0Ag zU#HSGF`8K)xjJ6C+<1XJQq6})O;2y-a|!C%^Go=^G3ujBV^kK6PtJ{%TgB8L?ppc# zK;hUDs?77BW7A7*sP7$f9e zQ&L)rOGJd6oqY%ziBUnD_Ee{vQP$dd^>$pw2#ORb;W&(?D z@iB-nJOffjI)F>)nrufj(_p?XxA%`0e6&{M)~#D;p#&W}d%t$2?kbd!Gic=FXnAsX8Rs-3}XgPvy!-4DGr_|pY3%Q_GL9?p1wX0k}V*(cp zA_y2cQX4X?rMK~*AmFb%++7Y=o8NdH96XqE_vsB%(tMAN`RXsyC!1>KsKfJ+E1uJ7 zUXDqxO352i1hkst>eWNXIO-9z!_&=yiSAGVPLQFyR8juF7=~}W#+$s&b4=Rl&*V0@h%E7 zL12`Yhji`K9UV(DO79wIckfM<8~7f$psVPdZcmTrmLlQKyD?ii?Nf{1sZ7}zRD_~X z@5*TD7E~+EKZR0cZj-YbJ%gH41{f0qXgLD$S%#oANiEG4FAt=7@1Uk(K2#*lXFr8) ze^W@{NrfW5bYD57l(19TP77T??7VvUGK>lggN}ltF}6}SJ&wb|$oD)Q9o_b})66K8 zrl_S^Sy|CiBG5gQTlCWi{+^B4R@T(WRS)0%79t$|`(1ZpIutZHJ-qPe_C+oqk3F*6 zVy0*2gFjj*u^6pu+nXUB!C}!y6->x3!EV;Y$WFr7YA(Q^vIIgHP1pm{?So~*qdw3~ zptTYJ{+|2$m*RDKKq2w+d9m14nuwg{`kzT|`mopf`X{m4Hx0bK%hh~;5JUv)IUR1ErqYM5PGEV~&BR%Y<=nG454mrm|!H5r*# zaI5g}@G{#;KC>u8vn?otD+(W!O?fUeZLN-L5_8)+kL*A-S`)n*4ZaQD&*VM2Ynl1* zT~GYfE8bO2XLIvxp}iIMJ+&gJjnWDT=)oFCQDG6>5$TNM><7o-GXI&--?*->ZVS$S zc6PQDT+MSfOwzfNbNJK!JBwkGWKa&%n6CFOcUjgrclInTD{HTIrSgLF+JrW{>E~nN z5S$QtAb<@F%-G*AB>NhiI(6z5oDQ$`y4&f!v`}?g+c?g-5Nn-EdR+;5=zJ8a;hfrv ziAu2xUy){JrK4q#hUzh$pfe7~unA4E?d_@B8a z$m0))*tE3Q9X~_-uPeV=&kIYYtDVf`2WFz(T?@iuVir)>j<1!?-j3y8QY6T zl(-Bd(2O~`U!wBNCyJB;R-#y@V|XEA@`K?7WWN6&7M%YHFAO{C+u#TCa`;dQQ24L< z6SMpRHEemPu0Dx;(Q|TMsxJLu=Ko_UTVtk-Ev29hFqg{-L>wS|CT7oXS~~hC`%*GP z$3lYO&(X)BFs0-d?s0a5_|llb=V(4skWM{ji+OP%`{R~unXu~}WnnYk0^I(tZZyJX zL9RmkuA4`;Q@5AyMOel_>b&6Pgr} z(sh&M$@q_-Uz0hHKN~eSVA%V?m6WO1RaNSK6dtl(3s{Dw^o-4Gs~uyK16o+w+B6Yu zf3Hr_{QoS>yWB;&`V3vON!2K2XVFG&$>g5oD+mQ_f71#Km-!_IYCt%Ex~pqW_RCM_ zrG?4KO#XYLoF6!sL(cz3D~+il)i;qF zM2|L=Le2hVlH+e925=Ry{4GBaH^M9i7sq=~29LV-FLw8D5(-uPjKe!W9O!F-SBKh|U^KvF|TRkUeOOI<1+0Jf*& z$)onX9{7bVWfjktu73$P{vHQM)@zc}da0nz`Ff(z61fJ5E;I)J&pKo?{{7KAS=hx| zLqchN-e#~%GSdJmtpJ)vt^F}&>HncA{|od}x8m8{9=BtlrTui61xhB*{U5DSF)^B% z8&F_e`n`~>32u^VDLq6;rP6j15S-o`$cE7mwcr?b1(5A%fLzWT961rUZ32LfB=@f` zWk|hz5FGYjM9@AEieRb`FCe^C@11;8iBlm~W2nl(7>%7kdWssaEabAAVvO&63o#1` z!ag(zw5^`Q%goA>4L342<}~lQc~4Su8T6W(*4BAc7Ck9B<5Wk4ioOrR+AJL9clGP5+Gr2ZqoRcQR{xx19b z>AIpt7|d8ZYm1ic{T=yLn^G%m#%03M*cR&1OZ{0406g<`D~CXm3*kN+C`Mq}`+Vf& zX0uNg_9Mm8 zP<#y3IC+9#6tHy~roMQ^C*e4$i_Qsoi;@ysS!$NA|AP($jJng&Y* z5u|o7AJ-fz8(trR061-+!Pi&t*O#Z5moMjS*7L8wQ+@gJC0eV@#?B5k9uY8#4;DKo zX_V41;j4~VU9<|%WvMTdW;y4&J2PiicJ`-3kNsaiGPik;I-St)KzxHXYxp>&@~i!# zexi~3-$&+uJl2Mz%PA2T21Z8bzoQ&3NEaYzzU1?>{5E%Q{!bK++GD6e!x;7BJQab^ z^ZOV7_uuAo9*H~-O=z_j*wSAywTiP4YU@=sCtHN5!su^5SzLjChWO7#4GNeM*|q?e z|8c_h7`Cm}T(y5^qkns+BGCMzPZVs#4PxTuo}}dDLW};lOCUWCrNYI|y;FdU}a z5p7f~EG!`WmJtL6`(gWu!?y2bw$vGg_p`t($14p%+YIt;D>xT`+Cfp!1BuPU(ovG# zlaHskR%Kh5fEc$lgof|YuOe(Rf$VYHT6u|Yqy6X4o<$?eUShj<4nLyOG{CF%fU;9x z|76nHX_v75LCgr)8W#uWrZie23Mq$HnjvUhMa9HGcpG-9@LIVJaJ}{WKV`ik@r9f& zIk)5F*6Q|+Gd)lT83c3NbA9yaT~kw&%<6cC^!1k3R>Q?ir9ddFSdNyMfKEj@5RVCj z4d>CS-2&!I$4jl&cdn5420nfkuhaZUB`D)SQ?F}j2+$tBX8_-qg+iqjI)wreUeSXO zkVjI&>07E@uUA!84uherqplqfMdTbaEeW6f7e82YkWR1I;04};la#p)($27uknZQ| z0XlS)lwu4Tc_8!d2f?a32xO&IKyC~=cXT2MiV`G%cIJW`$3u%;c+!+wSidU+0$xY+ zOITP;PADyyA*>3^tV;ItzP`RPqZUjkEW*hcM%*3Ux488iu?LqLl&A^PZ* z-?%gkDpVBUYUnhev9nuu+;d&(y9bK{l2~c1KL-klY@g{ts^@Yzd~H_Qkx%4Pw`(}4 zSgYT&g?tRIh?ZTW`m?(4;lCP72YGkY8u78FQ$`cWk8i^#5VQcP#>3&W@4mONHd&ji zoGy{M%?@QV@Hrn}lj;f%p2C0s9}Y_peIYLg^uw49oLCY9Kilz-eiN~C_rEGR|LsWr z9lSp?aB>d**V7FYhvqOMG6y`_A7J=p=*@|S?mt4@H!-=F5J+7x@&5!qS>rqZIqLrc z`J(vqtrxlyIn^Z-`yoYl+}ZeBXN5p;rZ;i^7xfV$Tkv0Q|KH9muckh;6MGtYVY zwx_^+vkcO|cJc(|WPASuawaTi;rO2W{rm4Bf5>l2WsM1Db}}?_P17JXP<=Y6R`@vV z02f7!Lb0LFmbQHTJ0Y5&CwX<*dLDHxfo6bvAbzx{zoF)f17{{Gl@(G|2Nb% zfz#b5Xun%2t0^@(+-<3x9K=Pl`UT7_`Aqy(qLw`YP4m%4 z-0kkO;)5*1(Yf4iOnr)NwUN4SvfOs+6i@1re(|~NGfJiJG5l?yMzUb@QzP3j+dJOx zght(2m_wbW&lR)xk}e4$<2%}@_O5dOpnExrYE@>6X{p=KTyDJAa%FfVDpOtOu222n zTc6E76DlsEri$_ud03&P_Ydy)kQ&_D;L`3P<}DWaCh_6)_ADr0*MnSnA9(7YvGVrt z(vH%7b3144+)193;t2Hw8+#QVH6lfgnT4O_SB;sTB0Mp*X`SvtlPb>Kwlv?e`l)~39KeaRF+NNZ-gREA(!lkS>Pcj?r-V$Ppk}t+Qg{K@ z6XQjfD%^ms+XEK+tk|)ya`)ro%wK{1!7eP05#T3qjE2ov%-Wdplelwy&i(d~>@zrx zN}V?ej>(w!r310{i}t%${cWD4qGGH%)E}9?^Nzp&XB#wOJ7<_)AieF&s4VV3^nF5c z`#YD%a)qn$jjz`krT@AsB8yo{fAU2o0rJy()3ag<&dfLY9Op3_PqH|&6ouzv;TSJ0 z{VLmZ^NA~5T?p|Rg@vd8PWJ|??J=?gpeycgAmL)#NpU`|U20S6_M2j{d-3Na2JWAvq3`vp)Oj|i>(+sS&eC>yC||voy!b;4 zOZ|kGnz;Bc5W*?f!eouBynS-*1icIh{Sn#%^K8rMh!L6rn%BP>2ylIs#>M~G-T8mH z;y<7SwiCEHPO~l&7t{c{FHJdF$h*vXl+Dbb&p0D9b7{(JT^*1CE*V(?aB@}ve#0}` zB5qrKZGJgN*5JCT1At};fHS8u^YLnBXQHcM-y1K)v75bymbuKTDjqCSJ~cEQ1Yjh0 zZXXEgLC2(;MRXHZG?OkGnc7TLLD|%2w^jHs^~sad!NlAOXf`=l_5@$L_(6g0*DrU7 zB`&~aL2DqoQ2^Z)0v%hj;pmJ7T2e&e^%`J_oWFm$D-ANT{(&q-@;whU9|h2f4p^_d zckdGMI%WWj-U9Z~5?Yl|(+v%ATsBCbgWV-aDMG1Mr|NW}UqQ{?y$bzB==9*<(6G`w z3EM6JUUDG=LHFE&vKtA@j&6R46ZEQ}T0-U)^kpljL*Z8#dMC`GyN+749N1qpl~?%g z=PL%=lUhPmC`^Wbtx62uQv=H0Wxa0aI0u-E)&-g@6FN4QVbKTS%IWg)Ty2)+mY?6S zMiF6wLSPG$$6S~U5xZ#+y1N72uPNq_jZPCmrNjZz8{I7u&uupZ@B|V?9+0>3Zn#U2B`k4TLP}HK={x65lWMM<-|4KuzBygZ3JFBCmlIRp?q}Z6# z_D`e2Fl;u3E??wk=c63+Jl*a<;!S@eWAj=wtlxh&#M(g%} zc0eh40NQI3JZd>nuNVq?0K{+tZ@xNNtBGdnRTTx*EvJ(Ny^yd-py*1bLbK5I>ua^^ z8nAguKD$eCv?ry^0XDF5q1{DF^!SMG-1oy{)JlP_K1--d=u|sqLft#)V|YyEs5KkG zc6!?*iI%`8g{MDWM9aK$e5Z!Io(?CcrHznugx$=m|lG&{o^+ zWOJx`#Btjtg-1j%2psPBYNNX~Vg6gt<;7e25b^71Wzs_{(q#Pncc&4ztEX^7A6&iH zqleg;o$l|y&c($wQscq^ z%}hUF0*xIVm&nM-ib_jAKpqQiK>+}D8>goYt?-J5hKE~$GAO#=cv@Us{3a{w)uY2b zmY|@Z`?j_Qx%FJuBYse^F`j99E(I-&3h&`V=to9Hqz;KrjWgX_x!5PbAY(u`*9V~S ze*mkVA{jyk42U3b7(t+fOV_Qm{q*V6doY63^z<{(Ve#%~I|BjRLyVyU{YzF>%QAc1 ztgMZ|7}#&j%e#Ac%s>Hbq|%N&C^)#-teXrxJCsu{KmySO4TN)KVHp>p&Tqrh<)-peA?*$^4ya%5eE-%2*<&k+JOJhf->{j>LX^@~V_^5s$b|E6CLbDUR*z>Q#%& z3JjEgZeZE0Me^5FESopMh}22XC`Um(G6#k8=elKa2GG6=um3gA9E_}}zn=de=lnCt zeW8ogda?!=oElDGi@me6xQGaXk&$uMbN`C)z8~~prhySvInH5%|G$cZqqOiow*)FS z&MPBV(X@{H^q*A`ACoBf+5;~YL$hC=R`F?9S675zu4bW8TR62=gO8xIvl=%7-Bk?U z(SH*}&AtQ#1lf#9ACcMF7+1)Jo~5U!BgMcD9wWtWq8H`TV;82sv;enm^$|GY`uiOI zHhTw+@qJA=h)vYsCFpd-xN^lGM7VvY^O&w*zpn1zvk2i}2F%v|*LMn#sQE+dWex`! zm{ydK@E;SR3Crc#gl=nNpj~jsT7w9o4s~s*8+zSaTLSSPn44pqIdkR_L}MJx=bYQg z2?;?ETScj^-DSix24-e)Fc%*o!ie_`$UadpYxI#dF?kL}QVJmzR76S4~2}?`*uV250CME_$^7a7QXAyp& zntcR`rTg{@;j34#zMxRAAW;`lR9qW%*I^MBCeKnxY$`SDM!x~PSY%{m3OIKpcJLpQ znN>255~I4-I52SI^y8E!RyQ}drJW1dBqE}sr%#_b1HRxX)UO{wTpp=%poHNgSC8WD z>Kmc&`o(-_Jbd*5kaFgONL&(9(qiDJ;0`zM+`&QEAMCp1X_uadY+eM$a0eRP^Ym*g z`s$61jJ`ul`~wi#oA+g01O;lb`wt`N-6sKt@H~Qq*YT|3qaWu?yOV+eseDf0y$&tb z5@6TGz)V~w=DG|C_c_Eq{2Bm{`}doVyuGbztb?COQ8fW`8k(FO0;S~?C{f?xFH{GW+Z7$V5jLh3AVr&C%TFO7qx=9yleqwORZ;+SkF^*J=$u!k8_1z}Dh?qT zfq47&?HfTaF4y%bCP6dU);`g&)Km)S&n(B9DC zTg|JgA^=Ij;5G|hAglMT^AgeBIUsH|#|q!U#OMX;p=)z(GH73=}uH7y3Bud!)a_N=ZqH?nb4(Ln`!0 ztH8oUG!+LJe>NMSdL5a<^>uXU6Y=I&RxU%7{G236>;=v3KZ?gbzkiQF2iSnCl$H~O z(C={x8W)j21V#S1JtU>3!(yV_R3r4E1;9@lSm=U70A{ZzNA))1Js{g6hi?)Vy)+Qr z8;%b7UtA_sv6&WoarrJ_2JonvHn$a%BJo_%Kb){KQY_}V0@VLUz|Wah#23##p_jZ3 z5{g@J^x!ux5Pk^>B-RsESHZUPe{fzpd)j)matNGY!SqmXpqxO@# z2N+NG9hZY`IvIq5o*dvPpZobC2#AQ9 zK7GP~UxO$tBJ1q<{P}Yccy#DrI|YkNOGkGX(Dx6}Lm)yKPuH1r-~r4&^Ww#eqO!8) zaBA6%5crh<9_abL1JE@Heu3fbI$)Cz-Q8CJMLbDLB5O8ch2z}|?N`;zT_q+RMIdkh zZBZ&%PenzAR?+>(dLe%yhrY8z-TImq}6%;LQ|A(N=T0I0o8LhYY{|n0hDL^L)c%-2S0+0j1{;B)_v4M=i&6U;F z|Jm~;b+U9DuH6H70q;|d!938 zv~b!jKLY?mfBNF3OR6py84_=V%Uem(e!dYa@5lN5!NTOPoBiiGxiL`YgG4Ol7)D85 z`oVr)`QVO!07_x>cmMsCR8yv)`{w&!my%5IZZ23{MZC{@Da4p}*=o{kv;y^WQ#n*K zs5R$zxes=q!1?p%Pb{YlN7xu=@QPDF_F813FqQ@hN-zy~Mc;4*$92ORQ*e$0EldFbwaj+2yp|2iR8!6H7Myt^N-H z7OO&6+ggi8<*T8!;L*<=iUqQv+@rYNPd+MIg@=Rc+YzE~xYZ}3U0<#JWjO=7PjUT? zn#eRg7^x{P!1k$=#rWUa?cu<`LPa6Oc8Sugs_|o2#?(Q}Gog`eW-<`APLO5I(q-^OKH}Ex+i9di+&@OW59RIEUCzELUoSvX0+8dy66UK> z2(g;}d_8yfQRWx(GDWz;Nfwg$)*<~7xvBi-g9j#BbJgcOTHF)j z43`&_Bag$3R;@qDIkS3AV_)0AO^w_8UN6s5IOB{EI`6Dn}=alQO zz<3w=nhV;UBGyg6jSN-k2>aL284|GF>ANSlXx#ivL(t@kyTfX~s(6*>{^R}bmOrV* z)v%#&Wo%g2uJ3A@uD)&B57)^rRePgZyKzeT@u#Uly>z@BW?EV$7Ys1DjOV`ypJw~K z5fgXGTG@Z#AoaA$i(BmHq;>NPv(woJ&H`tQCo%i?=vC|eIP4RY(lNyl(2uJ90|1M~ zjDXJ{Exq}F4`X?thY{d@19 zn5@j}?&#muHygy~_hLdiV8wa0A!3Jxg=;P!4n~#Wz1A2K9Y#-&T8@dMK}1c1fUR7~ zg$iD=Oo!!etx&5Y*!<@2F-R68Dfec@Q(&whPvi!9>~mAskC;pST`iYcAB1QVz#gR` zEgTQvX0h`y z6Tew05xz{p=ibiB%)r}~t48XpYS}@N0(yeKQ--?l!UV+Cd!AhtXS@t|9L$Ef47q+R za^s9SOn6keeMw2P^LJYeM!k701jw!bzK`<@l6 zztkk>NSv5&k~>0-Qh%v?R1q_CI*V%de*f*6EtD`9<8`_pa4uOsvE*nTkBI7mdcj?| z8KOlyXBQl{{lX{B8_(qn;;@V65_22T=)y@9 z4AP%I;*ECOp8R}jO6oBW&-Y_v=K%(a-p=A{t=WxNf5Txyzq`Gxxgb<_S6sm-(+Z0& zSEQK)N#-Qwbnal|`s-=IQ-~`2^;-zqcOn@p`^uIM=CgaY?JG&yDt~$Rt37vMgb6Zl zKX()KDk}%PekJPZNp$V{^-M~_S>;2C!M3)GG;iPD;be`f7#$gF`Y7>(@&f7>9qZNV z?qPixXV5b#MOjJsjvvzQS%-v5q~KdQ{wr5TMn>K%FP^=9`*ztz_HS<-BFF2u!Z38X zFNog6;XuWYmwZ*!X3Z8bxe@S2Y^#+-McB(|TN=H($~RTEy$c|$ZWqkMNYUmzdlNRz z{>W!#to-^9xAnA?6a*kRdnc!-4GqF?<>NKh8{9ReW6#~9#(jZK<-GS?&z?I+0ku!h zIz2soR*SxeLBzsjG&D3WYle~Wf!XG5!PaUIGxWHNhlaR{DvG10sxY66!d^6e=r8@) zij4S4iHi&rq-VQ!E9~K9ObPL{m$WoaU0H>!v)T=+%BIG?3Rg0+@87?R^Ir*(v2zx) zeIX(#dBLcdK=(BFDUo3AO8QO;YE_BH0lXOkpDDK3`&@VmvL-~DO2uPcO@eQ#;GJ+P zeMS}*7H`GS1d^@Ron1T-do=^P!&iu{lXpy;bv28@Pw-?Pf&!H;o+7dn6_MQvBBE zSkCKiU%&Q1z6UvxbO<2NrU%_Ehrmlpr=p{~NJtod&*#!5>RMvW3Rg)J?#fr6Q)Lw$mk&=5`U#*{ zc)sr$iHcjC!Od%t(b3TwE&!`91Ej_x43P-^e^k8%RF-KMHT(c7D4ilHrP3hXC?N=f zlpqL5DM*QQqewT>AreX|A>Cb~ba#i8NaufU=l#C*{j+A)tQqim?)!>!owLv0`&zEt z9@F^BKm6tNwT!3d#l!Ba*RDl+&{9xbd+;*$__7>w5MIHjz_O7Kc-FM5rlyvfpAVT7 z)*@!iN#5qa{ek)fM65e3ERiQJt1D}BBY%joaB&+JXJ%N+683j?eEl+SSG&16G33(A z2LvEL%J;)hPZ+0#l05!<2J*Oc4LkAAmCy|0t{RJAZ{;sa{>LE zxOuuJ2q>`ad*rOFtN=F*^EHn#Q{IbGKMHvst;iNt$@&ynbYH)L836f`2UljJ-F1M_ zoSaYU$J;1rXg-EmeI>SMw0naDFsK$;75$t?W9lCr&d*(Tt6m}A`$=oWdz-VUA`INN`8Hie$nFV}#@}#}vm+X@#WV^UH z1jQS-0iOUQuzimTGHp_aLKPM@?D}-qCb1N%SFSZ+9c66w>ST8V)0%ec8g_l8%bmKW zz8`utS8YW2$qu%rHBIsMzUQ56KEq|F7ok;FVA0FGJ5KuoP0|OQ&{nQ0TiCDIr;^P7 zzcNnMHos0BCsQd&5;lpC5&B!y)2C0{J0mCZqGMvxV2KZ397wFJtqrX?zjF1ck=Dhm z-OZge8neGFKAoH7^!Q{rKQ|W}4Ed_hTNAhU_m@_#@Hzf%wFjU)UIpJoc%4!nH-me4~Hq*gQLiUaI=SJojl$1JJIyyfr9636p*=WADQytUO>7wOl7%(ng z*Kj?3iGT9&`j;1YW}Nozbe7ybHmD`X_}n!vmgoOPAB<}7m-+S#@7s>kXt)QbHLY9m zYdTbY%OEMJec!NON{ElYDj*=xaw#_zO7vIu$hkmylSTDX>IWWi-O9Yi{Yeexss@M@ z>|x3uIJNjRH4Lzx*A zfN_w}eUEf*e!dr)s$Gc!_=rzm)rG*6 z69!O28v$#L-Xl_10?6rKWyq{3Z^O;;OfW3{LFf!v6%t@wiryqKGBx#AKDWLKN(y>Q z%B|l))1DIEA~rU!9UQ*xdOLe~h=VKQ%*I9t(yu*P97rw{>e*uNIH>6M>amq<7n0n? z@vlU~8kZ-l*uR0}gnz|?>eU$uV6RQkC?ue!)}7y58_6baJvrX7^^57xp~}fjfO5sw zpXHUJ5L5sUof00;#?i~-FpFqX6bq7&kh}*B7rfRCoBau~aB!r7%k?R<4NcJm3?Tym zg=DyW8}r4B7v*k8cZ-UOUjH6q2Q9@7Ah2%l?7V~Hxb|p?(y3D@FQ2=&w^!v~Id#qE z%=4n@yv8*==2mJ+O+)`E6Lxx>UdH%6K_N>i64z3A8keQQOEgri+kDw9}pFjBs8DtFM95 zfaM}-0Gyx%6cpNv{|12H;@W%oICOo+Y=*BLsVioAKALUs{ERm#^TE_ju(ial7oUpaG~2M~jn-%N(pT8L;vpprQ(f z)*S;JSilS8M?D9xL1g6tem1+db`8uYFJWRb0K^O`Bwtwe5G%LA4|Wa&OOVSo0S=7- ze5iVW;J1OV%m81@!NH+bYH{NVK20z%c%>l68ZWhMf-WQ%$hn$ZTM^_Be&-PS!Cb%t z*3i2J)z?>tlYPtek#ioRx%tnQ5R9B`53!0J%tNd-zeaxJW+cGka_$^p8Gc((Qqr`D04gKJME*za65*7I7`e*mx5(;aiSIL91vYo zQ&Ta4_bSqg0U`&?{(IoxX&#Tj-k=h5A%VOTR8pMVFp`F_*92Yn*}&)%2-UiWKx6LS z+uLga#m+XUv5Y{C6~}K$0Z_N^J?|^C;AVLRl_c&?0cg6Q=ea0=ixEI%jgS$5xp)Ll ztD)56p7*p&{fi&L{GDOqHJ6vQH8nP(qk1Y!C^shpyd`f+hJCEzvKuLQb;1V?-htJA zZu?_9l;lngBXoJ0w|wsOwI!PCld~{%ULnKBtQAtpMUTkNW=8rm5O);DflvB;>rUR} z-w#7~Qlu!NCEh$XHps%U_1F9IQ^;<8GvFb%)Gsn63N;`jmBe`>AqtvZvtU8p44q<| z`Bq%U+jWPu>cG^WnVo&x;EkCMs5O{i`gYAj^K*N9R>}f~A6gOhz_;}SU>ivOe7g!N z{xFEhf{*3YZw2*MFA!WO%55;fOV$U#W0)rv_X!}R&MYjv2eVH)=!L-vUhcBT0+eE8 z0-D;|9#kH?IyngfQ4~6OwDnU{G~fp-X=cU@xzKwUU^+1|YS3*WLwig4V=$y|vz`|x zw9v@8Lyd+Ywt+y>_rf?iFaV>yP7>crCuf77p65Z1(|=>hd{NJVyN>dTad3z`|0RM; zW9{wiPBGIKd>nF0up~v;m@1f?>qLGR1mdDwiCLI4anV2p^0J#tB{wRvFqFrFFYEuCV z1NBT-*TBdKX`vv!{5SXwZ0zhlp`l8ju{EPhqEpJWp!o&eOdul>L4QQ6%7J}-eVxm3 z^I4LFr=GSTf~K@oIP9^IbzKm&d7O~~uLy;5b;V>ma}y$StHuHsr@ZaB+Oy+^-d@5W zvWpbEs^?#M9cT#bb&EoKe!SnTFPe zHMMWDu;Rq8nGe-_VnRZdGe!v~uf-oRhV%Hp|1k0pqn^Kd;hWu`IiJX94&a4uNtk|#3$BHQ1qHqO z;I@ZG(a8uW=hz)szg}O*1XgTyk_nVUID~^g zPdRL7e~areU`u%=kodtA7Mk|bBd-cB>;+|IZ4jzIi`WPX3C)5f?f;5!UwG#qqEir% zp%n$~NLiqm6~G_Ep%yM~B+E}o0|t-jP~HvX=rA%eLP^4?qCO3^q(Tp8cWas!I>IfW z0fo-=Wyv=`p=%jZ1*N|E@VAjFN6u2K$!h>5pz6W<`Wgyo#sVl5c>VhM_*{mgl=Ke~ zwEHN&0#oaB=dZF>HK3-GwYZXZBM$}}q_)qFX}K0kQYV~=!z~`9oPV0LyL%hIBR0Ut zSUMu0C8#vveUgS!o-{?C+gpDhy0l~11^19w49Pmr1*}Tq_!w zRqcAPDPLy|#1%v`TYgi6ji3L?(lQKy=AFQ3&$GpR&(GIV{|^^n8G3QX9?;yi{oLJ~ zDuoH5va-(z+H#Sr2w6r81OTU;UQj}_lpimrjMUg-eiV0Sv9z@w#@jlo3ybM?eEoUZ zg4(P3&StRy=+Nbs@Dgs{T6E@K+CdpkP;UiV-Y)Ti>C65$k%0KjkVFyUJGV`r@vfV=PmR5ilVyXxbd|&DyX^ulv&8uL8?CisH15ay zK9VNIxbf^*y=R;l&TQkR*`aYGVs-=K~nVSj4dK7fAMt=Qo_<^48W;VtKxp6HW1 zZE1spGg~454|?uj911%DX`Q9XW)&BcEO#fjG@PA)di1vCmc(Jrz)H4E-o2-;?BzvyAEwB zPGS>S!JJR9^p0^7-};8cG4>Kio)P;Hrlq zAm#Kq-dXK(mn=tv!SBjUN5|3eQda`5j*bqg>=&l@n~@RXU&!k7204d_ayov<(4m~% z6O0pGkw2#%(vn*6CWu*eV|xtX05jSMx%bI+7Hx|&&yg7uI=!VVJMD3%P99I@Z2oOs zjy8o(vc9Z-N%_^Z<}`QYDtIVoXq^A_Z=Q-fgD!-Tp|>VACcmU~!q>66a8Tk&wWQiR zZNR%0D>v(V8Ys-%5_R2I92?4Yq*eSVOuDqR#1gEifHnCx-ke3BGV4OCeh6)3mpeQ{ z)mFcCDH2`nXk#8)%H1HX|jes@P(ix52m!D{Zt8!RTK#|6BVid>ojV zM$4i4H#>h`zB$Tnyi#C5F3}K+t;BlYKHVqhu0`bn>)G3d+~5G28b?ORMP3CT2KXWq zNTht1r_gNvinNUE!}0a?4epnlqF6rnkzrS4TcBiDr>4XV#e;!XMg7+U=wxPRZ`peJ zYo@-o1`=Hq3XlWC<}4L*J{Oz^*vx?+FdrNd5JpTW1s`h+(wXvkR3g*s!okN!N;#A- zb>FSJ_@~O9W=z}VhF?>9J`pfygv_u^Aq5#&31~4)61Ac7!>5sk(-S4BuTN_u>UZF)e4>aZ$TpUa2!Etjg}MvGzL^d&b}c+d-nh7 zIcAQ{JqCfkYRa<@fQe@dU&?v+zwJkldy_24W^Lgrm5`mBuy*^%2syFaZNvIc_n;Pq zFCD=Be|w;Op;U&18T4d72Ppl{i3wF6snUu!LgsQ_;{}X4i|b!V9{dkCEuQ*g^q|mlOK-Is_ypT2geZJvWl~un z7fWZ*{D%U4-PO+_cykD5b=ZXtMXHihCL%UH)xjOf#Ggk z8I;C#LD93q>eBXYr*t{AOX0^V$F~_pUh<4o#A&#&-P*MY2~(DDzxVJyUNvTk zARf}LEq<@X)*3M1_Ija*ZdJ(`V23!NWeh=K;T%EXy~gC<{%fH}QtA=_hi9D5g$F&2 zmPw8}q^dZ%S`!}VAU?!p-MNQ`4B!)!bRXFe8t;ho2U1Y9wDcnDEL)(2N@wy{8wGR) zpk3_G+P0OBv||17I%ZaoWg5xC%-j%Q1mE6n5EVtq&I`L?L_{!4P#Atl|MOE!3*Fx@ zSNaI=Uumrr{LVmw+CDV+$kK%TkT3Fd28};Y-w%#;G?d-mE|<$Lc`!L&V7d=HkN>NC z>_JkCqg8PVb)4<2Dh7-h%>Us&%!HnG&`xDPQd3oZ_n|Fpej#A~4s(PXKf$%+8Cot3 z2Zw#&%f);Xk}j$}dqIjWV3qf$+A}T90Qj==&Key*I~wArJupz!`^S&t;^mJi1YY3W zzi-u?4^lO^rtCL=Fa4w}PqD-QN=4mU zq9Y*VBS(>wb5LG%Y_dt9VF?buhpeJhv|Y)5m*JY7{92_HVINX`kGH$UyqCI?d@4X# zTk!0$<12ltySz*K=UEQr#32szTd`T2#dMyVOiTv-zO>snG{OpD#jZeUz}jF&f0+hi|ZurLr=b8%y8Ffcs?7_1V+2yN>=jPhw~xR_vYU{XOdI z4*&Qu6gbOipOcdnJ@4+f9U7ttePfkbZb89A5SdK_4E#AYvM&sx?x>H z5BbHU8q6|1-?uJ;of1bio#(0I3)eQxbcJNgQL{CDb3Mbuo`0SxDQTU0DrB^6Hp?Gh z#l?NQby(KhkWQUMt&^%5wy9P1qNFRTA+7Pg+sRKBoV96uj{6UGHzu)v54!`ZLA#{C zzZ}hKYBEgZ@YB*}eMJRT-=D|E_H3Kb#cuDO`ODkEPDbRvP93z5;{^0Qsr~S&FQKTQ zU<4mq-Pj+>+G&*s4x^!y;& z*(|-Cbc&uBa^kH<=`Z1BH$p+xQ!&qKYG&@;5=4d;uJ~2I>II+6-U{B!wkJ+4{TLp# z#M_!qf~cE7a|J*3n0=ewm3%?xIqA=5nlaeS9UX_SH!-(2`)=IFbairE4Gt4e@&iUX z^U)wnsOkf}QK3W)*NMu{+<7__KPG9fXWUX%ZVC(i_SZ-mPRO?r1QGmTz0tIpEV2ze z+`iJ%&CNJISG+t+3w83R^M%%1G|vP*Ywx>SDB`^6#zEbTAA7S@){j-Yi^-v1a}7{= zBnuJoJi3gU5pcc>EBR@~4Yk^ah^UEa0*FOBBG#L=e2 zMZ=nP9oo%~WK8@`qk~_`SlCxvtY3CNiF>F0bQ73JUN?{xXJ=;z4Rf2@9C)m?hv*ll zX9DPD7ABjIy+KM`zniTVe!NZRi|0O3$iaqam=g z$KP8Jec=8Q&htsZH!B2}2%fi_=XdPtKjk1HB{i6Pf7L%E1nH6H<>iG#*xuh9^#(dC zm6)D$g#6?ka16}y;^*_&qZxe|k1*IHt^^+b=44B{NG;|vHZ**9a5~*qw5VUeHQ;46 zRsOa+hP^3F;tWTv|GMWcXk@sTJd06(V}ytt3oyN|1hEG$n_cnW zXZlNW73%30-L zN5EPZC=}XN6|*T>rti;4--caxM@Jp*1TDgxH55bY;qHfrLMbbEyKp)^OnCJ1pj>!a z2?*Uo`6qD59qn)uP`grUUN}-izr{ZwpmkL()A-}}oQVqO`73a;<#gTbI%~aX{CZw^!rknM= z2e4lEM{${EeZmzeej;fY6;6a-9~SIRN8Qy}Z2YQ2h|C$lVH~n}{wF-r>z zF6(nQ$XwHGV>z~y1qpS~F$m#mHqAQEXVqAmw+(#u3+JbNy`@+o`qM4_Iw62&IkW&} z04;_Yh@?)E1||w{c`wjVFsiBZe|t%-L#Q!zb;bM@9%iU5b-L}ZW;(@1q61*>cHR=( zw{NbQkB(wX)G}TNuPZQ{IcKl52k+1(VtdDdysf6#u zD>0J=0pyBlWOVdWF*{I}#IGx1g<;t&BBu#aCNLp+e+%|

    ;RSsvj@N=N04mlLfA` zEX>Vmo4Mvb3B4QiMp#4yBRp6xi~X%63TYS$xm^-}$`-$~v(n4ZbuI2K3_@d97Y;DH z-|8gt-!&Hi3<@QAmm;-VC0$?9o}(_@bQD+W#fw|}TjQ8@@*fz_7!{Bc<({djoX;Um z=F5R&KEiDKO;OM8@v*VSFB5WIJ34khK8WCdJou215V!?q{@4DP7;`yh8AR7;qCE$L zypeL|$H#xaQHyLZUjGr1oQzw>Z`o5}d!3w;a`=_HvPK0Y|Ai~_J|lIGoCrO*(C9X# zn-7P1`ba2LsEP{iMv?gyrI0iNRVt8yK5fTsvh9ccl*H=A+;m&S!%$y;-bWAL+%3Iy zW$st9pqWleEu`q3BykBK`KM#nG#P)15t*n^sKurmyP(%6hVpu>Wg{(Ras~g!JL)P2 zO=q_~f!x;{YZ|*Nqr@VPTWqYhwia0!nux)VYPYAs}Nd*y+=C(GY(VfLOA@kB_e~1UAiQg;(vJH)TD0Re9 z9lNnmU6+?SL^E4yM2k4EZ=g{5cr?MuRR`^)5)pUdDlaOYmp@x~x?^YCQ>Ehg)d z889t5(K_v*M^4eUZs;vr)ibvY~MeP*#ZUejTk2(^DpUxt8d)x3(WG@9tX&U zkX$F7oq2nPe3h9&`_K?s3Wu&QgfC#08v!WX2qji}M#d%NPyk#H5K%Cz@PTD$Yd+}- z5)V|z3OAfg#$wLz`fr;I-}|#4k=OEOV@%_{j}JNuY|4WnW;I^G&2zg#u=m}H4lka+ zdKCyJV)IoO=a{Gr^U5SqkNd~h7qFMRjU=aQ$wf_X!i#1f{HEJ@l6df5ctNQxH2IVD zb1y&$FIi5~yHAW8O*gu3DL$Ck5RCtu+m&n;`7a=XzZ*or-J0_;0-Tlc>zWs*->35Z zviNT)Ci&msd5Odvl-K7Jj(Y&lZ!%4O*!-l)uWfZv>)yxjtPzcE;{4L|x!0|j7~q{Z z)YiC??iEo4Lnc8zPNX?6|!n68?CS-NXnPSI3}-hcuF zsL9!X5Q&v)=tvy7#UzzKT+4^@`u2@uib|Zys5R94<40K>3#)+y{*uN7>$(Qdp>GjZ zlv^`B*S2ad#8HUoq^GwR;8)*^s;!ytCMia+#NHfiZESd7K)>dD-34B^!)AN;j}M{6 zPREKn@vq_Mhum4L_pPRyw+)&QVNhEbQ@a0@^MNju5R|pZ)gEyWo}+3rM}2a4doT`b-{P8kJj^fcEVSwPPAhzX>z5mwsT1V{lfvRR~Qd0#NnPNO-Wt%j#IU%GSN0Js7Z zX^izNSu6<6{iMFc-h3i7L-H7!cg#YI@yso48%YAP?}H>|&z zVNg|1v7c~#27f89Qr$eR9rv<_bqOV0k-7M>pcDB7F`1WSK|_Dme5f(2bJ=2C=FF2B z3h^>Y z+S3oB1*#*<2Qm=v=x9>#+ga3G|K8;C=^@vTn*pPAUYe*QTRMx8E;bZkDvS5tkx z)MX8*hT!WJt{2@$C@ae`Nk*bnOC3kIiaA!ci#IaS?%hdzl;9(X=yzh+&u{?Y5_COB zpP~@KlWlF4Q}XcQ(5)^o`%4Doh}2!1t(s+g7>JdXm6_S5D3+7V*3@rXHumk*sKYvC ziI%Yo4?iVn=W=r5LnqN`dGesZ`T~udT)p;v+m8wi)VRm-4O)*~LTw@8Imm%ulUyrw z>c|`^bS<;9WLm=GE7q%VO~3F$LwQB(fAy-ZMSE9z_P*3|(!kIVeR$}r&D?+)cqvk# zw*M-RzZ$&wQPJaBk77GJwJkOsB?7JE`MIv_ zagdtvpL+zXDQO(R4XMGF_wLF0hsBoK|JiBGU{*7LxWlvOH=r|mjAK|BcOh{r=kgLB zVYWyJGag#~Z#7*T;Febmk)*W-sSy=ZBKX8`&(Q}^q%~pU&rOU@e`dJ%H7{>_^JFVs z%1@JmmrOOdh;9e{t=IqaQAZML_w=rMoGfx+OR1+99sSHmK0IN{sq*bgp&ZUfz{+iC zI(cPq?X_;tl8Y^|uaTEtQr`cdWh9Jv7x2awDWXsyZ+s~QBi%WvO?dGkcINccZ?h4TZ#f($O&S+l$K-e*trQ3yy6~lx6h`=ewB~e zPSzwg{QA7~j+k(et?bVAFln>bW^;LKj|Y@g4e24f1){O>HE|&G=HRYZ0KM7R*torT zkR-7%;A&D*5+tYcCK5X1BlWGVva{T`m~P!aJUycLUex$lT1lM#%?_HR(KTtf;BSPe z{3^8iKa|?9P25n+tZQO6MjnhpI=h8V#*#-rT$BE6s?yiq7pVTs3qk=DRTF8?PN!28 zB5plw>YQSwuzY(*nB^LDN5PT!0X$k?e;*5p?pkAW^QF-Hsfvjp$*Xy=oss#OK}_t? zm#?Q|oP!;%dh6&coQ%;SvOC+`?@L1X-rvu3A$6=h-ob`y85b_a%C)6U_~vGi)dlo) z)IgQwt{?*j81gri>+Ht1#q|I1X%$^l@K7-+A0mj@X_0tTKP*stFF{V# zSZimwPgdn-AkdwG(hFBvhlAGLwL8X7;=O4Q*h`oWGo`l?q49X$1*z zy+}b5DaqLRQ%1cwgPer)2D2tzJ8-~<_=o!D!XDpw9}vJSDk^5~mM6?CF_u|5l1|S+ zML)Xc=_cWCn&EG$%y7^gX=)W!*#5h3oljVlN$npMl2 zCa7NOBSy%al3mL@^?Ms{$mil_|5VRqxVGt;#dIw-FjD-3fc zL>&l>M7X>Gp{ywKZ98VSy=*obJ3ZO8(1Fs#Yu^#Yju1C!`<>1am1C$4lQ!OYZ?Wel_Y2N8~dxVzmy_?AQm?(WH}U z^bCa5)J!~$S$SG^|33;d9Ql2=IV=RwmoWI17hg_FNcYzRb<9{l?3WL}qRv8p!paq~bD9(_aiV zUS$rPtc>?3d7$xc;*b{;7N!8dXg@HZ*o_0I1<2=U3JMAn|CgSv6yJ-#y1I&mdliVo z3L2q!EUZ_Yom?Ic|42P%WhKXjK82UDWqMJBw!FMNY<|p}%qysHW}Il}3P@Q3xLyt^ z!iVKiP1rX<1OuLkmu`Lbob5RseR>?BaZr;bRO!VR;80 zR(-Z>|D!+`7Z-Q7MFW@uly6}4<+w|rIH!mqZ5a8FBCfmY9Q$$M$=J863+x1gOnzUK zQuNGJ+WY#5exSWxh}?aZxb6;#1s!xBH=dg~T26z^Z>v_^RG zH*Tc<+#oY3mT*CF>2I=6S58B~p2COvnhn#_TOrz*!3(LsOgZZ8)8g3P~ z5b>2}nX(m9rWYoXf6y|@%Nn^em#Nz9e-r(oN`zh=t>zWkF%(Z1S9P~GTfB}ytgG?n zN89}lcHWG2DUmYkrbUJ8HuJ~!0h^dNFFpc6(Xp^2HAgA(lFHNZp6)6~x;tD1-6_k% zA$|E8*fl4+GO^!uW#s8fDUp|7{Jh?f%aeAlzmUGOlgT8e-n=#mPkWnq$- z#!A60-3hL9w6L_4`7u_qg&L0e<`a15gW>kUU*eyt`>FV@#6IF4!370e0b7`o<+l;V zEmbBxEBih5A%5_5hy08oJTVa)vj5%Har{Z~BWd;Ff6DNvltu}eI;R8)lW(c=xI(h` zrDY-<-rW%s)1C8Eee~!NGy)o7?o#&Ga5XiDUkRjZY1kh8vV{AL{P(CjS)-m6+>Xde zpm?>m7PIQnF7m|1#ij7K2zivHuZXjAOiaLbahKt6Y}8E_WlD6Uv%dR#nl`!0&cyIv z>`YK;U+AeCz9Hw99GWp4U+%F}VOl}s7fr~VvCw5K>P~qpdI7kYWZ>Y#=py*Dz1<7` z!$a%$)7ig~>qo~Eq@?LXLqonkK1_;=j|_}0E-^3|A(3uuOgm~uL0+j9wA-wI&QJ@} z#4QzT^i@yyGt5@|H4nwvN}DZ(2RMWYwTJjKQt1Z%Dm;9M1kFFg0mLhlWt^3sed*2S z^2p}ul%o3}bx+)8yM{!~w8ByiLr7Xm`g8oIsD^rVWg3vQ!o$OF_S!9gpu51bN0Mck zY4m$hX+z@{SyMx=O4^3KoxQ>Mr?<@q7&j+~LDt^#jgx&g?N0SnlxX6z9wv`d9LbqL zVV-3>AY&UMc3x#=0!818Kq2)$-4B1hD_d=)Kc5yZ1O3H_^#39vKRCa+vj)^h4(aO^ zo0GqBRt96%`=1|7EfHBOWaQPFib``VyC2`eaWvc{t58Y^phJ<1IIw{U5DEjsn+SXD z%jeH@z$nEgGSX>Y8T(qr=JC%LMHN+~jOW8d$HJC9 zviF(E4VcRfQzaZ0)T8G9rdso<(H_5o2xDz_>iT8AeQ+?uLP5Itdan5@sFfO8TfemX zjRVR9s7I=E5E(hIwD?t?94c>LoQ8Ow%s-60#Um$Z?RB?-js;9rm5O^$()x&`B#*|! zAx6svQ~8LypUlFzm2Yo3L&}GU?x5MzQ1SicpOyjxwtXd-)QtDW2~-KivO70kTzm7| z>RQ?67gfN)I)MjZVzG0P-ELnbhl7UT4dBN69<0EchKH|18JVS)8JcBk17Ho90Q6SN zU|Woq>RJT4sER|wZluUkgmWAOa(=KIEGT*F-}wY_nYI($^3Ayl_|f{fC#sHC#Mc`WzkYg} z7A`1wtfuA-zY1#=u>^2(a#BBOeoSLl42()K97=BV8laqQ)loMM{)h#C2=9VSmsjMaA_ zbN3z~?6~5>Xy<37d-U|rQn%=yGWp_R(zZm*7#aRha^KHvJ~=r$YUt|AdA59OF>S>% zD-V6@bLt0A@+sp*5^c?)=EpWHV;+WS={74fXon|90WsM*IcWeLlJgqh`A~vvOO3dk zYK)uoEs;s_!IKx`Y74?o56J0rLRlEN8FBX>DRPTWA9P|+c~F`D#)l(f3PB|Bx|B|>5;=^_JNDdHz%J})=a2VjK3 zy0+MInC7>D)%|IYlzY0mcMJ~aWEjafwf(L@s>} zkG<(~%&(YABz1Zgd6)aKW{bvkO1oX`rn3xQRM8_G`Kq(5>}&&M$Ms|r<84ldxt`w_ zOG})Z+WxyIbJ2}C^Fdj{Cx=BM2zg6YW0Qw?CdS5}KxKeX7Qv3pnDm4C;qmbtP%Ptw zY;U1H!PO;k;_kxWYj$h7U2bScyx~jW_pc@xv6p@S=9CAkCLK%h}qxw4jnWjZLC>K7@WX7k~eR+W{Q zZwNUWfF7v&pmsU8Iwx)it7+ei497SBxHSHOs$Ul1Zd^<*uxaMC93uh090GQ@LbDCD z=?3QJL9l^80Sy^3NkKsu03GbF-U86%i$@&@l1W3bc{v12Auta@n9!;456TOCSUv#< zf{TYo3TXB=i=EL#VF0^_z&Igo^@5_JMu?Y4r399Ah~*9tR__6k5r_isfc!WQw{fDp zz}q9}S3g{Z@(jFdP_OsbpTfyx2t-FP74!c7od?7wx_@#N;hi64!Q$5364xp-V9>6t z2#C@DyVKaU+HZ3-Cll{>2U+dXi??mwfVUpiaK3&UTq?>1R;BEu!JzzhP=OrG9S8af*i;&{MF~zXW-p;NWrXOIy0OVESZ4jpzm`uO?x^oJc z!WNo&NIyYl*a+kKdTUBd)M*<9{?g6BM=b!CLj9< z1rHuf#McEM|GK4-fx%mlHZ+3%5|%jUbTs&aM4 ze9aA)mjsh@qk}2*eXwhkne6%to;g>bAi=`Ik_5FS_?&rzm5CAflp~uM?3V(?WWgBd zxwUoJ4W1XDJTK1W}R_>C*-f0fWDL zq#m-fj=j9o8HTJU-gjf-kqnBukdgWNO#fbvLi@W?*S^ym57KzYRl7Zf|doLoSE~@#Z}|8tNtz z_L9Vcp7OlBIWr|D^Z+Rq)NACUs48F2q8l0-B1bOh3J|k9FqiQKbD_qjrruIZooAJH zS3v^M2)1B^BqWSu!;_QLu$~`4LJCgFYU{V(0|^E(bpjLQ?e$R}WQl`L0I+7O!ZxR~ zV?K3t_3ix)9r$pl4%O{MEiEmPqnMFVn(a@2v6)(2LIR?$K$aTFgwRmQBCNmEC*kor z+k1Mjuhb z2XZ}})9GNsiC9sAhjP$VjjPc}fdNwRLe(NIW{0@v!I5XT+IJbY`1_ok98=wUV`F22 zZiigZO6-Bn+LbJF9Wh!8418GFMlR~aRq3!12zG%+aBvqC782rOfITQBG~Ih3%dd9+ zE4wZVCw!yl#Th0XjmSKMkJ!7Ck`fY*+c5vpOY=it3$2=D?biBair0T_jK_)DUg6L! zuh*+Wwso|kZe$2VD0S2L(o%k4xS&wrZw+tN*!Nk1keJvTE)1tqQc^;B!7?-e4>Qy8 zQuc3pHG#m#M-;_y3WJwl(g-)`4FW-$Kq{Omu!ckHun3kj zh*>OB-9hceOx6#so)0V8kOiRxsmr0ux}e?qz}4|Gi+hTSIADad{o`Yb&Ceg$^S71W zXUE4^H!;GJ&mrsNjdr&IkbZV?@iJf)IOJTY6!-lbw-n=0l3AL?;0q25^f{PvZ}0BD z4+>I<59JIoLih)BW+Od#=Ay#}GoKpL-VG{$d_OntY-~seRxn zS~o%?iU{o>eb-5ySC}W4W)ykwE=<07vZ?&RU9g(Sp|{7y-37MN*U8C3^EQX~mq1X( zitR~tK@ID+Im)cxz{G@UiwlV3s$NEy&FNdtp^!MZAcIBk^Bmj+B>T(7=Adv`J<*B<@ zU*kGb;Z6zRneh5`zvv&ascNwtEs^ms(8)!^)%sK{sqrukS(Ft&if`^^jka zHc>Zx)85kZxo8e)zDH^``8sT$+F&Tb9AzqflvYy0J3Tw)a$Ul4JKEDKK0Y*Z_WKv0 zOkQgtucjftwzfttx{IBs{ryVJlP9Q~Ycj=Cw(PL0Z~u2#mIwj26~aycg_gss(P6qXl@kNt^f894w^N3C16fs>`0t79f08RSI8OS*_|_ z*WeUELgC6v)OPLlUd4$@XSH9w@R5Yfw{HETajX_^^$`B{n7B!y8WXjPr^9b;$d0Uw zGOCgPJ*Ye6xH<{y7pEQ6ciMY>RubOU-6>WDVu|8 z1FPH-|L^jz7MSRs)=AwdI#trg$>RhOyal*~(J6W3_%B%*A~BTa<<+@ii1a|z;=LS3 z3GNj6B(H~h9(V_oXE?L^yD(P2RLsPttg2?Fd zF-z)FXqUQdr_G zPoMTG7ZUbEmkFmX{bF$59lDH~z{jdzw|+5~FsyNO1tAHm%fqjx8J3tWULiJOSRW~U zG?f}o1ozK3TDOJHl^H$wvpFdePa^=Nlv}N`EmLhnWJ_G!{Ec65DIbdN!$&nXk6@+C z;&9qN118N#p(SOd?cpunR^Rul`T@k7DOk!llcUOqmk z%1P=r$62?yRx$QND4YJa(e|dlR0eCun~*2gZB7Plk#R`E4KvkEK;5U06b^8~15LaE zTFG5 zQbArkQnP$r^eS%4`B~juUo`8XS^tviR@GeS$)cuyv`{M$UDrG*s~K!KQ-QBdNYCxHRwJ z0*1qhiXccBAz7;LoKP(G*i0Hl^w|67$E?1%)bChhIp=@m*UFANSpJzm-itf>DFBUC zfvH+&Z%E7=PLAynF)%o&eE6bxFHG4rFqCv}?()#Lr!c5iH}?lw$!}1mgTgFMz!MGb zA@PH+ic1{8MCF~%Vp0jYkzWA!cCG_>d=HYo`hCKOSlhJ&0 zjr8p7fQWS2bjxuEB5oYLFJ7p_z{AJH}VemB^2WK5*CJ!C}|-XYG*8mn>9NY${BTG z<`D%h%om~2GPf+iviz(y6egmTA&4 zl1fvR!PlLR;>wqA-$M63egykUc88Nxke|Z0fs{gacqKR05cs^b9r& z+Xe_p&@)+-$srWhx<8QtKCNhdd44U(^xv*r_=2qs+u^*Be!>U%lGc0$ zp69tUdgl2#CBS!wXCo>TwW}ckYF`>VJ0DK{GiBA#%BFQTdGUf?RJ6KXU=$Jh!qNd= zPXkn1hycASo(B_xNt(UB#}U)76bXFz3*4ovg+6i_ zyfbx&SA_!14*&=Qa1lks*^q3bP%xO_q>BkfxG!*E4B@(-G=U;U9dH5?blzbEZv?oa z3j;B?hO~YMFR0VBzZDk_R^x$jaDIRiaoD(m zI-IBpf|o0_g?Q$de<5~ybRdt!SsotZ5*(Zv`?D+GT&JNvgMxV<`0cf&8u69BSCH@$ zovYcj<9>Gq+{uw-mrC4;0PutVV3aJwhZ{ms;o)ADTbT}L;XumKt94`U4FEb)FqmIF z2apEJ3Rz=gM$jVoLTL_NON_0l8Yxh0dmJrZ2MB|nnOWZAVlY=TT{%S@DNKNQgy0cK z@ebffV})-Q^wOX*OMi7l))0egoHl$j}Z3e8zK`5S^Mk79VV)aGe=Y-CaalQb`MzJ zQrkASt8noM_{CQUtqtaf0Uiuh6+y`Q1{ziXS4x$mf?^Ca4>q6a*rF3jk87U%%p_ zxC8|JyuC4?+4~+spQMHcTy#Mr6FTjAekuex!OYwoI_j~C%3D7_%-WMp5yYw+iZL>U z=df2vbMWu=!DX)^NnmjRwzez7`LxjQMWHOmir?nsuvkr12f$+qiil{sSU}v4_E3_E z9>y!OEs_~#V&zdN7H4H8hV|x;g^!8S(ta9P+TY*rotjDregMp45}Yc=R#tDpP@wmB zwYFNmUhTqOrmBg_;xt42)2BY0H-Lp)Xc9T)C_@byn#e`0A;a-8 zzE|%-TMvDyS_vx;5y+RXTbct$PS^@)4Qx|wBh}8>yw)G5F%G~#3x6r>vOk36*zetTUtn~95Jq;vKBS5+gfBYqq zhoq`0kdzfMYe0OXVY9|4sHRmq{awOUN)gB9G>k%*r*FR8u29K0XGJOwxD(yVn8c(CK6!7bW>f0~hLs zH2&+Pd+2cdrVVq_>!VP4d0(B+venntjt=~ND_v4uWp#C>EaEc`bXD30dwRSI3%Oy& zePX$}4ckJ3f{1Z3@4~9|^}T}N?2?C%(`HJYrJ^-7G(M%KqVbiSN~8dTCymBYMMY)H z^NPKembPKdvUbwNvS@-JZ$l}yh|I~z__)zd6mz-91_i`xz&Gg>UtZMyTq+T|Sx21< z=}L<+7z03elufUag!uL&jveg_n^Q+n+h84+yqfLQ;EYR%pFNehm>KB!&}iRBPF6~>O213a{s`ZBa&_iob znP9Mo*dyj{&gOxbicBGH1h3EyfwpCbMOIZ^(vHfrs06AUwM;y)77BWH9n7vF%MDP9 zN=ytQWql%-q{wmyFgv*XkvR9BItagL;N}r;FzW^m^K%f+BU?@!E|tEU67(5nBiN$7cu}WMR{RgKW~ZSth zE)pm!?KZC2^6=zo?YkH9`9Zi3Na43`jmlOdJX|4!Gz<%6I<3dwvMTQ!ZQ`W*bhGMR zo4w_#@AsEV^V_LXaDMsbCQ?pSdxVsjKY`;;SxKq!Lgi7S`}=-y=4h;2wmIr!0=mTw zDE6DbCJF?Fm@ft{j+7eR26XAFouc{w;R4K5IN+i_J)(3pH8rj8>!fICxCi#&zJOqY z_xj~C?GogSOcrrK5x6`Y{HB}PHT?N=A|-N^{(pVFby!th*Dt&g1w>LoKtM!7QbIsN zLOMi38tE=UN;;HIr3C>&y1PqKKqVxl1f;tg&e-_8=X<_$uIsEn?i;q7z2{o9#vJ1p zOOcs>*yd%B%XRES?baZp?#~4&VN>B3-q6>e^^rkqPC|^-atF8({oV+5^*%XobLZ$a zbkNV3d|W_9!$Tst@jGy3xQ_KzvnZs6bS5dKW6MRL7w2gF^2q;^Uxy=7GhO+;au}cB zu7C$d3Z@gF?cSRqONwa5D~0&2c<)**|LDlbAISXAVU~bU2?!fKEnc}eXuN<-1S-%? z7Dr1T3>E5xL*N~~@=}Q%Lb)>YK}`6SnTsu|qpj`Q`uaL#7qinA7e&C4<_h8o&h@7< zL?%_deA8WXGQJfjF5}#YX4Yt~{PEtVAv9RHsz)Z>Ul0Xh72n>nO+vL^M;|xwm3S~}pTY83x_OVGnJHL>CGc~dRf163;Dgp^f2KA(H zIO^m9=Ps0V^y_$9*#k>OXqG8`Nhec;^Z=Nz**S?Ec0Q%+}vKfM-XQJAd7D*mNIy`Q^Ua9o^Ze!l%>DByUS|U zMZk3WiVX|fcS=vvhIRgDlJAr!AMAyBied6?e`x}`*qCJ2Kq#lvY;8R-vnb#FS_YN7i=e z(K6$+PaGtWMe+B3YPGvMNLuSL`NFieV4_BE>KxT{b{z+h7$d(&^vA*yp~}R#T_sBv zue%eIZ>l?nfL=wOEMD)Q9^K45q3!wtELgE%qAxI=VGX9+{VM!gx^^RouK?DiI_rt{ zR;kG8kurK9o+0H=U^$u{*_w^x+tBReSsuy{E?u|~1Go@KQUE%F>REySYVsaSL-%&y zRCr70C<&*c-R#{Cq@m|*1 zisdc*dMg$TC+*O(0j1W)3s?1ihtq=@8f4)CHsL>Q!IIp3wYqR)hx2Ewt6_<})3r1* z@>8u>A zC+;@_73F7mY;4Hv4O8^rzlMPfkI_LU0O~wY{gwbJLdYEmu8vjW($dnVTCGpk2ZPH5 z31swp`}^Mp2j75Ymd&|D{>JprL_XxX17aMcU&q&fvPK9mICSZO=WeP22tVk-_BpcE zFy<~TjR|7>ejsiMWmcpY2gow|!&(e!&myZaE@+I9nAkOVPI2r;n0b?Ok7blqzfupt zStfYg7=WUw8_>CWW(JHqRXMpvGO_w675UKRVS?wUGLzHtoGAeZ6Kyw$SZxnk5kkAU zYCIAmK@gK;z;Q)O=`R^#91`D)Y&t=-uis}9;ZLUq3NZXt*BvC*r}b?GPYsIiOh=Ou zX(JvvBo&lhFDAJL7Ff1$SK7J5a=1AC8NQrFY1q-pLEfe$%gW1Z$0e+FAoR>N#!y z4dj?bveFeX85kHA`^!m(tVaSIH~et{g)-fAuCui^&T}ZIJ}E+fj8}Ty!YgKCj{F*| zxx$fE3-St(IAh4?4#>hJ=6gjZurj3+XPb}}K{w;RFqA|1Q>Thj;xl4~mo;`+?R7^2KJ@;2(c2o7Mo^C4=ff$~;k_O97HD?U^iMsF@8>Xq zrte{vv)u@n>Aem4tUC&=Cm`!bLBVMq207N)H*`wA9Oj1MgBJX3Zmp80W@ade0%vEv z`Lb+o*JBPaRh}c$yGE~pd!H7mMM2$unY2`(QBYPE>zS-C_L!wDGaI|@#oDq0Wlh#M zLyw(yH&Z@Q<=qpLf`Lu#0O>$@s4;Jdsi~|>YG>UPxK1ZA=S$Dd4OZVYFma5)*doX{0e)tsw$W~?&DP#J0j*+ z5i!`YJx3pjg@N4#DFze-OdJt&X-vdD9vYJAo}@r4O`G~MBw%Xs-O2FRuWX+6A1FgY zeD+jSRC@acqnD4eEdi17IZ}uCN<@Eery!Q7>)l9Tt3?tuA+?XR;&SW%f`05NfPlI0yB{1H+UB>n!sanU4axDRI3Hr0rwu5j zN)5Ej1+X-e*m z@{|uBOpOiAkbk1G(W9D3_zWvhTlL*l{4x2qVr;_Ov3J5b1(M>{pIn>g{4q!V{39EW zsPqGH^PWWVVWXwS_SdHS`w2bC1jD+CSnfmh$V4vLZ#bH9@j%_verbr%-2BLTtU$W~ z3jxs*>9Zf-ffGbi*c%)ho0gq-RpF$^JITP%C8*0I9G{H9Ct%a2wXK74Mx9%KRv-W3 zE2@F>KvoK>45Li35mYaF2eZ{Skf9tL27;JxT*rwqof>FP|$nTvbj7~!tkuSH~-HxJJ2P_XB zUPZq~$-_eexv18*8yEVEMboBxTOG(54oE2k0_Lk)xf@A^rGP1tbuPNWqwUTQEDcZx&(G`{jLNGMqOI zWbT&1%U>x%qpjJSBTygB0z;wsEgxq2zoO->i^MN5_)@Sc~DRU=I%lj5<4< zilgxtjm64Q*_KtUxF%IG&B znYY+gv(Ps<+b@ULNyioizFH8v>cLhAy2ylDEMsWc;2I-|b%#Cen}$5i*52NS!>2=l z8dX7xidXohM@KK}#rUK(L6HG6ECHKECe5hUqN0H((9d4c<($7OHxP^#7H&K)NCrY% z1%19ptZ*46$qt5_0^QTA2*3mxJ;y7?rZ31D_d7lv)LGpl?oc=!FYsj3R4wp-9bWWz zp55@TVF2^vUiPNn`+`c`Pr+CS{gKQl4yB200KpxAX~DKFE{+B)x_5#Xhav+Ok|5tz zCa~e4VEt1E&ZG2^v@;$;2=Gx-$b3$2YHkhyKUBbUf+DC-ZZ0F7hkTCrj82y_IBtou zOJlXQw(5fA7bFRo_gy%l9;{hm#qwIb8fN&_YXWZz!&$DNQV{YbfYlU!JU4fMpIuRt z#nE3HEU3f?%ggJgOyWCpaBu`!iU@7tR`#2VO!>Fnu6?dx84}Qvkk`zA&>eA4pEM^v zN0?OcFL)JWv1mFO6hFHu`V2>*{IWEr(9n6SeHLO&-^uqxW#)T#`dSnqba^5k+8Ws_ z50phA&Dza(;E#k9&gcbMY0xPvG2guxp%{M~?k z4`OZ&%O1Zvbag0IDCFZtG;Esmh}J)Y?L$n=W9P1Wkc9~NcT1OIXTNk&>Z6b;pq4`R z46c}^NKg{l6`!kpZzg3v#4u|wP6%})R`~LKi=Q8@4|hr8yskjjPg5D1h8G@A*3m!t zJ1Zj7py`u8_C~1ESoRdO6V}nMB4KJ`4V35zK#agWG{Go7*2|1|V(gfe0ZoEi}I?Q-tomf-Dt5TpH%EJHT zePkkYpG|RG95Ebg@7=p6WX=}*Z9(UcdSB^sRMrBqkgCq{76PR7uE8=x)1VDg(8lt2 zS+5Ob1<0$aDts(tR3U#~G8FiBbFVrWdu5o2hiqO1(t=2RtKF3`33IEgoCC717mB#| zD=6n^H@K9Nl0+!hsjMB!SM;q;D7wbqNv`wzLq62dWjp6PRN`Cr>Kthp*N!2wQ)qzY zYxA!Z9ry&K`kj(W@%#eJx}jtNTh4%G>y*bL_+%S7*V6cF3!=^QgH!lQt%U3G?aJroF&Mgyou12?D zrD&Ip7gCiVQsEkEdzG2{CAjrXAQJWp*=blOQs94mJWzbR@k4;>Mezr(;lPqL1Jigx zCRwyDmwI@tz!D6UbKQe;C8O(Ox~keYuR!8hB5znkTVecd{XM0@0r?lIL{P7kDaY1& zT~Q!^8g14-OFS+o&FFZRkVp32i7YhtA(wvls9#h<-I%;yIxgc2(JiwrwS`g?!7@$C z4f)ohd1Rpc;Vmhv4IUa6nbQ0jV2wcRm9md?2RLUCF-AYA+YSu98fMKfCLlWwxMPN% zy<@$6%VmlW_JCm*6Zd0%_a4wckYCW#H^-zhpn_8uK;+~tYj!~J4$*HVLT;g&*y`_H zV;;DYFg(z%^Wz5YYdum$?Vw&i%-ncF+g;8*wicuKx)wU0PGq=!pT=nHGYXVYsB`uJ zo{r$-tCbFrehpdM=p~m?E>rpI3LiMs)W>wE-vm$pS)}XRcgC&iw-o78w99gqwR-7L zOwaD!%9U0}kb+H*O74(&AeJ3EM=nkIFW{rk246q@!7f5pR#rO47-FF~ac+Q=Bh`{y z7YTacBvh>X^-5sjFaVPo*m!d=sHnc!{;+N1qAo~l#cSe9`%X}b9W9V&sXR7k#GPb+ z^?{GB=bx{_-JHst#m(;Y-!0-*U7khg+6F{DpFdd-gW)hZVRCjd3&ZVjqw^Ll3nai9J|?lSh1SM$B}N*_=I?}$PB3=HQ>@;FZNO{GT5?>(7tL~ zO>92U_C%N*T$z6ATlt6`^<^z8YG$-^mn+b+$aw8BT0p zsl+mrtg^09OJSh^z(UQzT;%I@N;M|~Hg?OWSxTB)`0lAcC9W9^~>$(YA*-KAio9^ zVC2af#7A$}YFA+2(f!&i2JmYP$TKh;iYO>k=5v*`W~_?XGIYaKv2?ks^OslZmWcC+ z-=c{0)|)H>+}XQ&oj;6xZ;CL;e|4*LHhs1G;nV%dPtr(~pnEa0PQyiByHM3%$ySNl z5clHT&TTQ^0Un9Qv7?=1^a(-4%UDbuKB1D82Q2Ij9?p>26`czIzXF`OFmTVz;&sBu z*WNmyCnR(tRpYALU3*!J$?sQi7v1qY7UPw!9oBXHhT3WZbslx!A@^B! z)I=s;r9-mlJ`ckwyKmz-MMW$w6C&25pI#Jl^$yf8I`oK8%iM)!0sn*BU@t0d!v>pD zQY&oYPakohqRGa-vJ{p6Tyj%Hw2O)FF)2?qk8h0J>v-zuR(Ycj?zZ2W@0yNPB0t2? z?x%)dTk=l3)Fq}UAm?o#fJ8L}Q}jEI3?BU%!L&K7nqkAI|3DJ{Bsr`3Xt}JB> ziSn4^@o%ENO1c?w`*vs=MM53D^xJ?yE3;kiKRb z`oD}pR*5O#bjZ!K||;{Q+o_ScFwi3QHGTZnUdT(?RNcy5rt% zj6d6KdKen~-Khr~>}9J9dD0BI=n90-v5brjzwuo~KKQH|*= z8Ez7DW2?DpWHB2W;(Quo^!Q$>rC9 zGIV`j>JJ6wNy)%(Ts*s{ub0M5&5062K!<3Ahk}^>Mp0Sp=_eK{uaL6-%2HG6BR0a1 zWpI$V?7RDBGqeOBhX&D&H?B71H=+A}xjjczfJ9Xa8zunvw_w-m`x+)u!B55~YvUvV z?%EZ9d*=*<3{98DevHzaJpCX>X|zU+H~g-UneCZ~7jEqcq|7G{P`11QA-s=;jJ(oc#I90KqHj{B}+DK&`2$2fuH(Np6 zmHCTIR^op!lwM|5`ULGM{409z39DyySi^fE_=;3^g^<5^2=0o94HfdSDNX5Lgu_!) zsoPQUDUEVv)4t!F^+tazGxakSzYvLd?eq_8=+(@+zvpGYr)~;>KuJsjDxS3b7q$_? zjSqDVDF>*`RlcizCG|v&X|rc30Hqp~$8ymPS}T*6yZ;9_HNK))T%Z%J4w!g$0hK62 zjniLx<~N~}(+lFaD#)4kokh6te*PT@NFgDFyt0B6Ly|-ydWTTrD^T26=6gJD<6m(~ z{?4MS)uTwPn69JRP^?18iTNFIaLTsHAbdDEqDXA}KL)ANa{HYrnwc_5yp!Y4w+%S$ z(9g#isJ)uswV80aI=zr2YZIThwl>7!ut1G_eADJY<9ex1ETQ`Z;vTjNgy>1LWNp)yy zFOQdZ%KI|9xIewQ%x;7WBgf6x@^cd&iuk4&QfOC3xH!|$Vw@n)5O96y&%SdlTV&(< zp3nC3jXCCS(wpho%mW=q-?ZY|`R`xdti>=)3#53!uUz-p!68z0YN;>()JVkmdLY+X zjhEI9y~T;SDa?(N#oN8k7_Kl9E-!kLwhf-YZ}2g&@#L^jX2hakQS^~BFL}RcI!44= zL3ivVcg2vj@P26#QG~hkuKA7OC8MjG3tW01J+vrCRifUWAgL^HeNcQ4t2b3ry0w8N zC6)t2SGBvBHhIeCu0Q5a>7k0k_kTtjh2xAYe$%GJcApc{Y^Y8QNr=2s8RYgk{o`whlK;@>nDoK08Q1>vi!GvuY5$b1_d9u95v`}X3&Wp& zG0Cgyzi+hV?U>^&4Kt7Rd}h)XKra)>B8iIqw8E}LY@4Y=l$COWOeu_5(xrJPE>Y9IvEJSG^j(lVC9QpM{vuOvfslM?2Y5)elON*!g z+Fhpw3KlBa#p}25?^B25B(yqF>N&31pO+Y{_)epuR7uTh<9?h1c3hug?V$M#ebYtG$s?(t`%$I~@)gQo12k#!G{aUt?jCJFjL z)|HDgU>EZDMLJ&vef8bvhFQIg^8qFvawM`=dNn6+UsS6;?h)}39-wh{x;4V1BUzT7 ziP_DI`@!83W-szmT0$;fIz%>08Wrwk4nErAG*WUuHrTFWjQLM?%Q-1Ma4GkuEJg5n zezO0JK-@u*5`C)l09-1=`yw{R3^-h||Fs6N$Za{`XSB!Utq|-p)9Kun5uBRMOzp?C z=IbId2%!IzyDz?KtN$4H9gYpVtz)B6Skc>^6*Oql_3hjH%iUf1XFP>RQ$@a}^u)TJ zJS)_ayWErhrcqzg%D|5TcfidvkMbSb%s%thKvT4{s3P0tk4`>tb^$BxlHuhitH?$LqF*76{-1bqFM z@pqp@0ABss>@%>j9#gqP$w)o=z(2_!F_{^p9ZRgU^|57H{mQKBjlI`NpIQ03d0WFzc*~40vnyCR2+|*ONarnu z=r0)=w0({M$RRw|B#n*>0M-`*p;2Dr%j@8K@K%h!uDzoJDXHqw&`{<4i&P@)Ol5uN zrgFwycY_3Aq=sAyM4)JxZu_i)JT)~t2g6E8RyM!6=WQ?%v#6yd6Ov$WZcOX=!~xqO z-D4=q-Ik7B>5w6eBO`iv2<7)aVT0Fc+OM4s;TeOelI6!yc@7(=D+=*-m#lSVO~7h@ z1y+t|Azzt;1Tc)mT|>S?9a*#+40Vj|NC0VSYiF`v=+#4QQ8B%qbuDiD&3j1fot;U~ zpI=G!hAtkdIYCzbS^Dgc*paD9gj(s)t%;g9g(_bZ4aXX7eirmMHpEe#?ab4Bf*Io* zm1)J`rfpXx%zJ2sj zX0hdxVjMB4p3p^)%{J7;%U?)vb5(T2^?lizihO|Z0@Bdua!r{&p0NtE(23gCsn~3b ztQ!kJ>qbhT*HAoe8Xdic*!(pQc2l07g78safj7{TEZ9*-|Dv~2Rj!W>^0BPzD}{gN zXRuEEu#vBk@!N{7KH;x<^DeL$f5^F@Eq+k2X>j!VtA{_4$Z=o_rF(j3YXIQU3*G=y zHVA~>!uR&LKH+>h=&r7qVZg#4K&;&3@ZfDT4=9`2xUL4;*-~nXZ_*nv`s>uVjj|pF) z9aG=q!ob<5>o7AqDMt%P>2UqMPyJs=<{M17IPIXtj6gc`9WIQkJ89LwwV0el+RK0I z`|=vxwonjI&=LhZd1-!CP=SYZqW@<=?A?As;&rq$Khxh9Y&u?b9q|VU4vW8XA1H!r zBOt(_mIUahrj8Eu@Vf#{+tInL`-1j=5P&-W3;=Ba9IlR?+yJhUZ?I}X$@3f09|4M( z@jM}{zH#*_)FF8tm%(tT&2_bC4X^V#$k4Qh-1P;B07cy-UufN$mG{Q1@!Sp^*#ZqT*8P>j_tDY5G%}46 zjXeTLy-$GWgP^c3R1Q)Bsexo60J>!si8fp@WHWLkz|v$!E@;so4?;7G}@--+$qX5MV{563&@CXk8oz(pJy_1pzng}^lcmWQ`U zginv|NFr7E5J;@+&c!Fj4&{{P=UQ6iNVM|g$#g8Z`ZPB^eb-i@H%1u|3h*AcM?eNU zI%xqw;}|er0BwtOhXdjsqRCG-VVc5*Lg4)1GZ3j-E3NTuSPG@2IY(i}?_Va9p%jOlPw%Kpd2u2$d=Z5J z0aZd)KrA7_VgO&!!6HQi<-8rngKFr(Ag#*RJH}WPY**HxssO$1=I(|-lp6PVKVAEc z@Y=fdpm}`!3KGTd@JrM3Le6DZDD$H7&4Gf#kTXSxp#BfMG@8_m`_cjkN{!$eiC!y( z-HgGp(((t}i~3@Zt-6d&%E#+(P)*qHW!2{9$9&W` zxr_Jw<|s-{dr}CNCPtadk2^=l$I|S=W2HJtFFdAI?{Jz&X^+-x%JEk#4Gs=Qqy+^9 zsqhQ#1wKHEBwf3DmCkC^cI*fYyjG@&g7yl3nJB5rf40)p>78Flil$apRn0Alu)#ne z5O;I3*Q))ot~-t#F;TaUTsds^y|Ly|Scn;Vk+8C|lAC~rlc(BYGqwV1>tllZ7E`M1 z9zu6-vp1LexhMjy`Hn;dmR>ReKB3{PwLKas;GWta{su5R0sifr~zEx9?8=abLNMbBE*G(8}NOYP*UZKsk0bx;$~R zdEat&bhNNc{-#d*Gmo32R{@)D6N2BtLA=Mp^SV+)>rBokeoHmb-|cqZp_cMg+bYXPInmJ>%Zz=q9+lkKDA3h4M2e2P{Ik}hFL3Ik z9Ucw}0`d75=w~i{#3nT?*DyNTeNlh9{ZOi3&coLAodt__-N>I$dqK)jO{sC(O116z z)VC4Cq@WVBg?s8x^w4s@9-u*1wU5qTuW3Yk9wbtmbgiDhbyxy&@->?tAl~nS z^;}j)hPW$xf3)3GuJf_tF3ITh45)@JqP;}wcYxY5# zI0T+OTjAMWuL$lEkUzku`4=p7(UFl~0E3+7nbPH~ew9^V~x)JNYWZl5`n39FLmCyX|%j}$B97n z?loIqANFgs9N0|o&y`C?oE_w;yX;V06rHuy0R>mueOn6F0dQd|8x{YIf`W4A@vFL+ zjP#{B)&7LKf`V^s2KIkPUc+;Fadvz#3~VvvjhvaCq4C#m<6;*igP$MZ#%~Z36K7G1 z_?e!+CE%|;I@)*NX;Kc)%rFNceNLUd6is@1evN+*PA`u>t!%%XfdP#aWAb@@LH$q~ zvt}hd2gftehPfl4@3Y^yl_4HNOeyud!O9{bNzn6Y5Gd8I@{UWmVgJH{o$tGST*4F%m4=+5458B z?;pRwm|JPV6&wHz%HDErp*fILK-mB|nWUKjYcz%WytJg``^Lsb0D%_t_bW1M*Qo(p zzd!WxWEPNMEP!%U<91X6jFt?*Q0LUtXdH~&g(H83`c_t5-AmvVNlQs7=<0q3Y-lEk zjlnd_S1T5Va23R{Lf5Niaq?eQD?%hy|ElrK5&P~FH672ha;T|t8nj~pF(O+zMNww9 z|F^~OFU4%JX+UoPIMWgo$ZBlog4Xu()Jw_QO_uu8HH!6p9oNURVRHp?n*|`id4Y-f zZmO`LKc+alcAb}pr@GIg!g5sKH8DCm{m=XMgi{cdEhs(B&fHzi2n`Ag>}zN||1mgM zM(1(N;T{qBMS?;zwJn@~e%ba<+uNI@4|7-W2!@6u4fMy`k}(_9X(3TETT26@ef($>(x;tfZ=1+cwq4IoN(dQLd5yN`d7GQ8GDV;;Da2%J0w3 z=H*@H<;&m+sw^!07O))Dor@1RK+_{%y;L0LinO$J1U3KmH>!9ZyNnjBXy^uVZW|$$ zTxEdR$50;~9)8m=UIYWucwUE`SM_@;QvKk;Yx#mH!S+uR3UsL%V1y&mfZ=2g@(>j^ zrtm8)ilzD2&%ub8kC*H@=E7}km$|gNg2T+9__r;K=wS%}B0?{NxBY61P!($<_25v%2OFH+- zPHbq6{Za(lxP4@LZXdq7x8k_%c;8mdz*d`t@s*_rAfO>-T7!{cn(O@z&FuR~agFn#jcRN^1C>N*e_XKvCxi#8u~%3+Owfhi^k(z> z$3KDr#}9K~qu!+)&K;a!p}RI-UHWx38Il1@U{k3I!-krkjZFbLSe`#0Qq^%!hgE0A zpTeWp{a_*!dR`hd(<93CTF{P8PI5t1rosfqbvC$o;Xc{)I^RjYFg@N~(r9q2E^U~5 z^=17K5a!ts8bL6Bygz2mY1BmkWo%$Mbq^X{fxQ!fAX@BwWt5mwR3r~eb0zq0?tXvN zW549;a9edocrqKKmKXM@W1>4 zQTH$QgWDjYBK!AYoBIE|eXH^F4RMKyYk9om&mp=m)iGI(AMQW=*VF}5F0zP2$_BQ4 zY!BkvcKlD5FZa3^s6b@@8 zC7if-F6zy)q7Fr48xt|5ezIQu& zsYQPINsGd}WZTBkg#E53>8mYa3*9p5ksBm>eXA1J11{gNsmsK;qY_Q1c&y>dI*=Pq z;91C_pLZ5Nb7g73wICb!@=e^=9{JZPSg|U-)Fn%BOF!ukArXGC;``KSin~$Z@*RIT zzLRZ>qidI;i$vI)1i3Gbyq7<5jx6Nh{hgiup9iEr|GtvPPPpqwF@?mlsJAOug(Hng zI@%V}KU7N9pYS${|L2o@L?-7Y{bsk))UgY%<2Yc2T)ytrp|Ae&ai@Ucs;Eltl&&$_ z<){9c!|<;YGxCk9p4xh87-C`m@4F_@y1VMc=2sL4X*%qvtnFl(weoUux(2~c5w1U6 z82(SR6dIyV0(ZCYJ9h5YrG2;Ls;&7q8phG4-)Fkm)!HwgWi)EPq|0~ib>gRa8Wd#r z9P|?4`mDKhm*1c1V}EdWXao0}g1X+gb?54M&G_h1kEW+pZ&pDr%yzi!Eeeu(3_GK6 z8Vu{6>oETCP}<(JI6I3bYj+`(YC`UdX9S|&JLG%;PtAl zd|G}&SCaio(oS+^zQXoVPw0s`J{}3}kCMipKVF||*ksKP|9kJS?LU!1RyD^!v>>_JG~bD~oP?@Ms` zxGL6(sKvx(@cSYPbxx923V#2#{uZ&sHf!=<)3Ake`u%{t(BG8@p_VN-OH_}S8 zl&+a1X^a3&Pm+~8xLnH&EYXfnn0>{is9ss=^Wm)X1gd#?@~ zD{ksh)Krgm>7(K0R!99hkENAkS3ByDi5uff7Q*}G)*}k4E6(<6f{%Y>ktT2s3~hC* znd|~l3whmbap>+^4hk`IEu@sm*gdOkw}>q4>vee+qO>0xbx|GVE_s~Nzm=3D)88l% z(cJ##`NV{>LW98~mDHCn|5UNZVJmTM&lr1e`B#ySyhQuI6~hVZ0NX$4##-TnAMD26t4f)CjSJh?G+c7-at0yMWFnsQ}hQCAnJXr*tNNI#__ zFKo1JtKQ?$Tz?SOVW6Q_GCZ`!$g=c__ut^}b_h=rA$K@0>nmMM8QixMRFYPB5qgh` zd%(2e<(No^p{;DE-k z4OUZO^$l+yDYo-=x+U_DupUWI-%2BIjGbBFDIEG~+!-}FW^Kt#k)Vs~Qi3yDS+p^j zdBnT`JvBSSyK!=PH;9J|4b$%nlVSBOZc%xW^Ki7~rc{lcx4m1T{haOa@UU{wcKSP( z(eUtfw&oNV(`SvO3(p$bM>sOIW@GiohkyM4CeDeUNCd|{wfI zFRf<&%q?Vm$vw6Ca^Ay$_Jg5YpVO00Y4rX-e2Vl1;hh>#p`CNXtwvr z?k@JV{O|R8T#zgxj&?4*fTn zY3pWcO+^`s4h)Z&Ls(xyZ_s;HZj+0qxt_mIo|(SApa}yD84|C X6lF#sdri@ae3O*ebI}4}-M9Y>e56|= literal 0 HcmV?d00001 From 1e3a3b514e62e03e2296cb245ca4439dececffac Mon Sep 17 00:00:00 2001 From: spisarski Date: Fri, 23 Jul 2021 15:28:53 -0600 Subject: [PATCH 26/40] Initial changes for converting the TPS AE to run in K8s Created new AE image with minikube and dependencies installed Changed node setup for the lab_trial scenario to load the necessary deployments and siddhi operators for which there are still some issues --- playbooks/env-build/siddhi/env_build.yml | 3 +- playbooks/{siddhi => general}/setup_jdk.yml | 0 playbooks/{siddhi => general}/setup_kafka.yml | 0 playbooks/general/setup_source.yml | 79 +++++++------ playbooks/siddhi/docker/Dockerfile | 16 +++ .../kubernetes/kafka-trpt-ddos-detection.yaml | 68 +++++++++++ .../kubernetes/kafka-trpt-drop-clear.yaml | 65 +++++++++++ playbooks/siddhi/kubernetes/kafka.yaml | 110 ++++++++++++++++++ .../siddhi/kubernetes/trpt-to-kafka.yaml | 65 +++++++++++ playbooks/siddhi/setup_siddhi_maven.yml | 4 +- .../templates/minikube_service.service.j2 | 15 +++ playbooks/tofino/setup_minikube_siddhi_ae.yml | 72 ++++++++++++ playbooks/tofino/setup_nodes-lab_trial.yml | 28 +---- 13 files changed, 461 insertions(+), 64 deletions(-) rename playbooks/{siddhi => general}/setup_jdk.yml (100%) rename playbooks/{siddhi => general}/setup_kafka.yml (100%) create mode 100644 playbooks/siddhi/docker/Dockerfile create mode 100644 playbooks/siddhi/kubernetes/kafka-trpt-ddos-detection.yaml create mode 100644 playbooks/siddhi/kubernetes/kafka-trpt-drop-clear.yaml create mode 100644 playbooks/siddhi/kubernetes/kafka.yaml create mode 100644 playbooks/siddhi/kubernetes/trpt-to-kafka.yaml create mode 100644 playbooks/siddhi/templates/minikube_service.service.j2 create mode 100644 playbooks/tofino/setup_minikube_siddhi_ae.yml diff --git a/playbooks/env-build/siddhi/env_build.yml b/playbooks/env-build/siddhi/env_build.yml index 3ea2f02f..69511097 100644 --- a/playbooks/env-build/siddhi/env_build.yml +++ b/playbooks/env-build/siddhi/env_build.yml @@ -14,4 +14,5 @@ # https://github.com/p4lang/tutorials/blob/master/vm/user-bootstrap.sh # Project and script derived in part from the script in the link above --- -- import_playbook: ../../siddhi/setup_siddhi_maven.yml +#- import_playbook: ../../siddhi/setup_siddhi_maven.yml +- import_playbook: ../../siddhi/setup_siddhi_minikube.yml diff --git a/playbooks/siddhi/setup_jdk.yml b/playbooks/general/setup_jdk.yml similarity index 100% rename from playbooks/siddhi/setup_jdk.yml rename to playbooks/general/setup_jdk.yml diff --git a/playbooks/siddhi/setup_kafka.yml b/playbooks/general/setup_kafka.yml similarity index 100% rename from playbooks/siddhi/setup_kafka.yml rename to playbooks/general/setup_kafka.yml diff --git a/playbooks/general/setup_source.yml b/playbooks/general/setup_source.yml index ae21c9e7..5b5baef4 100644 --- a/playbooks/general/setup_source.yml +++ b/playbooks/general/setup_source.yml @@ -20,48 +20,51 @@ dest_dir: "/home/{{ ansible_user }}/transparent-security" run_tests: "{{ python_unit_tests | default(false) }}" install_python: "{{ install_tps_python | default(true) }}" + install_tps_dependencies: "{{ install_dependencies | default(true) }}" tasks: - - name: Install apt dependencies - apt: - update_cache: yes - name: - - python3-pip - - arping - - iperf3 - register: apt_rc - retries: 3 - delay: 10 - until: apt_rc is not failed - when: ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu' + - block: + - name: Install apt dependencies + apt: + update_cache: yes + name: + - python3-pip + - arping + - iperf3 + register: apt_rc + retries: 3 + delay: 10 + until: apt_rc is not failed + when: ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu' - - name: Install yum dependencies - yum: - update_cache: yes - name: - - python3-pip - - python3-devel - - gcc-c++ - - tcpdump - - net-tools - - iputils - - iperf3 - register: apt_rc - retries: 3 - delay: 10 - until: apt_rc is not failed - when: ansible_distribution == 'CentOS' or ansible_distribution == 'Red Hat Enterprise Linux' + - name: Install yum dependencies + yum: + update_cache: yes + name: + - python3-pip + - python3-devel + - gcc-c++ + - tcpdump + - net-tools + - iputils + - iperf3 + register: apt_rc + retries: 3 + delay: 10 + until: apt_rc is not failed + when: ansible_distribution == 'CentOS' or ansible_distribution == 'Red Hat Enterprise Linux' - - name: Downgrade pip3 scapy version to 2.4.3 due to 2.4.4 bug when running receive_packets.py on centos - pip: - name: - - scapy==2.4.3 - executable: pip3 - register: pip_rc - retries: 3 - delay: 10 - until: pip_rc is not failed - when: ansible_distribution == 'CentOS' or ansible_distribution == 'Red Hat Enterprise Linux' + - name: Downgrade pip3 scapy version to 2.4.3 due to 2.4.4 bug when running receive_packets.py on centos + pip: + name: + - scapy==2.4.3 + executable: pip3 + register: pip_rc + retries: 3 + delay: 10 + until: pip_rc is not failed + when: ansible_distribution == 'CentOS' or ansible_distribution == 'Red Hat Enterprise Linux' + when: install_tps_dependencies | bool - name: Copy local transparent-security source become: no diff --git a/playbooks/siddhi/docker/Dockerfile b/playbooks/siddhi/docker/Dockerfile new file mode 100644 index 00000000..0cf08ce4 --- /dev/null +++ b/playbooks/siddhi/docker/Dockerfile @@ -0,0 +1,16 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM siddhiio/siddhi-runner-alpine:5.1.2 +ARG classpath_lib=. +COPY $classpath_lib/lib/* /home/siddhi_user/siddhi-runner/lib/ diff --git a/playbooks/siddhi/kubernetes/kafka-trpt-ddos-detection.yaml b/playbooks/siddhi/kubernetes/kafka-trpt-ddos-detection.yaml new file mode 100644 index 00000000..77f19f68 --- /dev/null +++ b/playbooks/siddhi/kubernetes/kafka-trpt-ddos-detection.yaml @@ -0,0 +1,68 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: siddhi.io/v1alpha2 +kind: SiddhiProcess +metadata: + name: kafka-trpt-ddos-detect +spec: + apps: + - script: | + @App:name("KafkaSourcePacketJSON") + @source( + type="kafka", + topic.list="trptPacket", + bootstrap.servers="kafka-service:9092", + group.id="test", + threading.option="single.thread", + @map( + type="p4-trpt", + @attributes( + src_mac="intHdr.mdStackHdr.origMac", + ip_ver="ipHdr.version", + dst_ip="ipHdr.dstAddr", + dst_port="protoHdr.dstPort" + ) + ) + ) + define stream trptPktStream ( + src_mac string, + ip_ver int, + dst_ip string, + dst_port long); + + @sink( + type="http", + publisher.url="http://localhost:9998/aggAttack", + method="POST", + headers="trp:headers", + @map(type="json") + ) + define stream attackStream ( + src_mac string, + ip_ver int, + dst_ip string, + dst_port long, + count long + ); + + @info(name = "trptJsonQuery") + from trptPktStream#window.time(1 sec) + select src_mac, ip_ver, dst_ip, dst_port, count(ip_ver) as count + group by src_mac, dst_ip, dst_port + having count == 10 + insert into attackStream; + + container: + image: "spisarski/siddhi" + imagePullPolicy: Always diff --git a/playbooks/siddhi/kubernetes/kafka-trpt-drop-clear.yaml b/playbooks/siddhi/kubernetes/kafka-trpt-drop-clear.yaml new file mode 100644 index 00000000..0be12801 --- /dev/null +++ b/playbooks/siddhi/kubernetes/kafka-trpt-drop-clear.yaml @@ -0,0 +1,65 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: siddhi.io/v1alpha2 +kind: SiddhiProcess +metadata: + name: kafka-trpt-drop-clear +spec: + apps: + - script: | + @App:name("KafkaSourceDropJSON") + @source( + type="kafka", + topic.list="trptDrop", + bootstrap.servers="kafka-service:9092", + group.id="test", + threading.option="single.thread", + @map( + type="p4-trpt", + @attributes( + timestamp="dropHdr.timestamp", + dropKey="dropHdr.dropKey", + dropCount="dropHdr.dropCount" + ) + ) + ) + define stream trptDropStream ( + timestamp long, + dropKey string, + dropCount long + ); + + @sink( + type="http", + publisher.url="http://localhost:9998/aggAttack", + method="DELETE", + headers="trp:headers", + @map(type="json") + ) + define stream dropAttackStream ( + dropKey string, + dropCount long, + count long + ); + + @info(name = "trptJsonQuery") + from trptDropStream#window.time(35 sec) + select dropKey, dropCount, count(dropCount) as count + group by dropKey, dropCount + having count >= 3 + insert into dropAttackStream; + + container: + image: "spisarski/siddhi" + imagePullPolicy: Always diff --git a/playbooks/siddhi/kubernetes/kafka.yaml b/playbooks/siddhi/kubernetes/kafka.yaml new file mode 100644 index 00000000..ad6b6629 --- /dev/null +++ b/playbooks/siddhi/kubernetes/kafka.yaml @@ -0,0 +1,110 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: zookeeper-deploy +spec: + replicas: 2 + selector: + matchLabels: + app: zookeeper-1 + template: + metadata: + labels: + app: zookeeper-1 + spec: + containers: + - name: zoo1 + image: digitalwonderland/zookeeper + ports: + - containerPort: 2181 + env: + - name: ZOOKEEPER_ID + value: "1" + - name: ZOOKEEPER_SERVER_1 + value: zoo1 + +--- +apiVersion: v1 +kind: Service +metadata: + name: zoo1 + labels: + app: zookeeper-1 +spec: + ports: + - name: client + port: 2181 + protocol: TCP + - name: follower + port: 2888 + protocol: TCP + - name: leader + port: 3888 + protocol: TCP + selector: + app: zookeeper-1 + +--- +apiVersion: v1 +kind: Service +metadata: + name: kafka-service + labels: + name: kafka +spec: + ports: + - port: 9092 + name: kafka-port + protocol: TCP + selector: + app: kafka + id: "0" + type: LoadBalancer + +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: kafka-broker0 +spec: + replicas: 2 + selector: + matchLabels: + app: kafka + id: "0" + template: + metadata: + labels: + app: kafka + id: "0" + spec: + containers: + - name: kafka + image: wurstmeister/kafka + ports: + - containerPort: 9092 + env: + - name: KAFKA_ADVERTISED_PORT + value: "9092" + - name: KAFKA_ADVERTISED_HOST_NAME + value: kafka + - name: KAFKA_ZOOKEEPER_CONNECT + value: zoo1:2181 + - name: KAFKA_BROKER_ID + value: "0" + - name: KAFKA_CREATE_TOPICS + value: admintome-test:1:1 diff --git a/playbooks/siddhi/kubernetes/trpt-to-kafka.yaml b/playbooks/siddhi/kubernetes/trpt-to-kafka.yaml new file mode 100644 index 00000000..1c240575 --- /dev/null +++ b/playbooks/siddhi/kubernetes/trpt-to-kafka.yaml @@ -0,0 +1,65 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: siddhi.io/v1alpha2 +kind: SiddhiProcess +metadata: + name: trpt-to-kafka +spec: + apps: + - script: | + @App:name("UDPSourceTRPT") + @source( + type="udp", + listen.port="556", + @map( + type="p4-trpt", + @attributes( + in_type="telemRptHdr.inType", + full_json="jsonString" + ) + ) + ) + define stream trptUdpStream (in_type int, full_json object); + + @sink( + type="kafka", + topic="trptPacket", + bootstrap.servers="kafka-service:9092", + is.binary.message = "false", + @map(type="text") + ) + define stream trptPacket (full_json object); + + @sink( + type="kafka", + topic="trptDrop", + bootstrap.servers="kafka-service:9092", + is.binary.message = "false", + @map(type="text") + ) + define stream trptDrop (full_json object); + + @info(name = "TrptPacket") + from trptUdpStream[in_type != 2] + select full_json + insert into trptPacket; + + @info(name = "TrptDrop") + from trptUdpStream[in_type == 2] + select full_json + insert into trptDrop; + + container: + image: "spisarski/siddhi" + imagePullPolicy: Always diff --git a/playbooks/siddhi/setup_siddhi_maven.yml b/playbooks/siddhi/setup_siddhi_maven.yml index 8be739bc..0871d18d 100644 --- a/playbooks/siddhi/setup_siddhi_maven.yml +++ b/playbooks/siddhi/setup_siddhi_maven.yml @@ -18,8 +18,8 @@ # ctrl-C will gracefully exit. # mvn exec:java -Dexec.mainClass=io.siddhi.extension.map.p4.StartSiddhiRuntime "-Dexec.args=/home/ubuntu/siddhi-map-p4-trpt/docs/siddhi/examples/convert_trpt.siddhi /home/ubuntu/siddhi-map-p4-trpt/docs/siddhi/examples/simple_ddos_detection.siddhi" -f pom.xml --- -- import_playbook: setup_jdk.yml -- import_playbook: setup_kafka.yml +- import_playbook: ../general/setup_jdk.yml +- import_playbook: ../general/setup_kafka.yml - import_playbook: setup_siddhi_p4.yml - import_playbook: ../general/setup_source.yml - import_playbook: ../general/final_env_setup.yml diff --git a/playbooks/siddhi/templates/minikube_service.service.j2 b/playbooks/siddhi/templates/minikube_service.service.j2 new file mode 100644 index 00000000..17b89ea9 --- /dev/null +++ b/playbooks/siddhi/templates/minikube_service.service.j2 @@ -0,0 +1,15 @@ +[Unit] +Description=Minikube service for TPS AE +Requires=network.target +After=syslog.target network.target + +[Service] +Type=oneshot +RemainAfterExit=yes +User=ubuntu +Group=docker +ExecStart=/usr/bin/minikube start +ExecStop=/usr/bin/minikube stop + +[Install] +WantedBy=multi-user.target diff --git a/playbooks/tofino/setup_minikube_siddhi_ae.yml b/playbooks/tofino/setup_minikube_siddhi_ae.yml new file mode 100644 index 00000000..f3b35832 --- /dev/null +++ b/playbooks/tofino/setup_minikube_siddhi_ae.yml @@ -0,0 +1,72 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- hosts: "{{ host_val | default('all') }}" + gather_facts: no + tasks: + - name: Create Service File to /etc/systemd/system/tps-tofino-ae.service + become: yes + template: + src: ../siddhi/templates/minikube_service.service.j2 + dest: /etc/systemd/system/tps-tofino-ae.service + + - name: Start tps-tofino-ae.service + become: yes + systemd: + name: tps-tofino-ae.service + state: restarted + enabled: yes + register: minkube_start_rc + retries: 3 + delay: 5 + until: minkube_start_rc is not failed + + - name: Wait for tps-tofino-ae service to be completely up... + command: kubectl get po + register: kube_pods_out + retries: 10 + delay: 5 + until: kube_pods_out.stdout.find("Running") != -1 + + - name: Show pods + debug: + var: kube_pods_out.stdout_lines + + - name: Install siddhi K8s operator + command: "kubectl apply -f {{ item }}" + loop: + - "{{ trans_sec_dir }}/playbooks/siddhi/kubernetes/kafka.yaml" + - "{{ trans_sec_dir }}/playbooks/siddhi/kubernetes/trpt-to-kafka.yaml" + - "{{ trans_sec_dir }}/playbooks/siddhi/kubernetes/kafka-trpt-ddos-detection.yaml" + - "{{ trans_sec_dir }}/playbooks/siddhi/kubernetes/kafka-trpt-drop-clear.yaml" + + # TODO - Improve validation by for all containing "Running" + - name: Wait for K8s pods to come up... + command: kubectl get po + register: kube_pods_out + retries: 10 + delay: 10 + until: kube_pods_out.stdout.find("Running") != -1 and + kube_pods_out.stdout.find("Creating") == -1 and + kube_pods_out.stdout_lines | length >= 9 + + - name: Show pods + debug: + var: kube_pods_out.stdout_lines + + - name: Stop tps-tofino-ae.service + become: yes + systemd: + name: tps-tofino-ae.service + state: stopped diff --git a/playbooks/tofino/setup_nodes-lab_trial.yml b/playbooks/tofino/setup_nodes-lab_trial.yml index 1216ef11..135c9750 100644 --- a/playbooks/tofino/setup_nodes-lab_trial.yml +++ b/playbooks/tofino/setup_nodes-lab_trial.yml @@ -19,6 +19,11 @@ trans_sec_source_dir: "{{ orch_trans_sec_dir }}" python_unit_tests: false +# Setup AE +- import_playbook: setup_minikube_siddhi_ae.yml + vars: + host_val: ae + # Create virtual interfaces on switches - import_playbook: setup_virt_eth.yml vars: @@ -45,26 +50,3 @@ port_to_wait: "{{ sdn_port }}" wait_timeout: 60 load_p4: False - -# Start AE -- import_playbook: ../general/start_service.yml - vars: - host_val: ae - service_name: tps-tofino-ae - srvc_start_pause_time: 45 - local_srvc_script_tmplt_file: "{{ orch_trans_sec_dir }}/playbooks/general/templates/siddhi_p4_service.sh.j2" - srvc_type: SIMPLE - sdn_url: "http://{{ sdn_ip }}:{{ sdn_port }}" - telem_rpt_port: 556 - kafka_host_port: localhost:9092 - kafka_trpt_pkt_topic: trptPacket - kafka_trpt_drop_topic: trptDrop - alert_pkt_count: 10 - alert_window_secs: 1 - templates: - - src: "{{ orch_trans_sec_dir }}/playbooks/siddhi/templates/convert_trpt.siddhi.j2" - dest: "{{ remote_scripts_dir }}/convert_trpt.siddhi" - - src: "{{ orch_trans_sec_dir }}/playbooks/siddhi/templates/simple_ddos_detection.siddhi.j2" - dest: "{{ remote_scripts_dir }}/simple_ddos_detection.siddhi" - - src: "{{ orch_trans_sec_dir }}/playbooks/siddhi/templates/simple_ddos_clear_drop.siddhi.j2" - dest: "{{ remote_scripts_dir }}/simple_ddos_clear.siddhi" From 310845448433ff8c5fa6a871259d376b8a610b36 Mon Sep 17 00:00:00 2001 From: spisarski Date: Wed, 28 Jul 2021 15:05:22 -0600 Subject: [PATCH 27/40] added utilities and sudo password for debugging to custom siddhi runner image --- playbooks/siddhi/docker/Dockerfile | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/playbooks/siddhi/docker/Dockerfile b/playbooks/siddhi/docker/Dockerfile index 0cf08ce4..fd7e412f 100644 --- a/playbooks/siddhi/docker/Dockerfile +++ b/playbooks/siddhi/docker/Dockerfile @@ -12,5 +12,16 @@ # limitations under the License. FROM siddhiio/siddhi-runner-alpine:5.1.2 + +RUN apk --update add net-tools tcpdump sudo +RUN apk update && apk add ca-certificates && rm -rf /var/cache/apk/* +#RUN set -ex && apk --no-cache add sudo + +#SHELL ["/bin/bash", "-o", "pipefail", "-c"] +#RUN addgroup siddhi_user sudo +RUN echo 'siddhi_user:siddhi_user' | chpasswd +RUN echo '%wheel ALL=(ALL) ALL' > /etc/sudoers.d/wheel +RUN adduser siddhi_user wheel + ARG classpath_lib=. COPY $classpath_lib/lib/* /home/siddhi_user/siddhi-runner/lib/ From 12831f7af7a4c5b308dcee01eb0bb986cc3ebad2 Mon Sep 17 00:00:00 2001 From: spisarski Date: Wed, 28 Jul 2021 15:11:17 -0600 Subject: [PATCH 28/40] housekeeping --- playbooks/tofino/setup_nodes-lab_trial.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/playbooks/tofino/setup_nodes-lab_trial.yml b/playbooks/tofino/setup_nodes-lab_trial.yml index 135c9750..7f343883 100644 --- a/playbooks/tofino/setup_nodes-lab_trial.yml +++ b/playbooks/tofino/setup_nodes-lab_trial.yml @@ -11,6 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +--- # Copy TPS source from orchestrator to all hosts - import_playbook: ../general/setup_source.yml vars: From e7ce39c2cc946f2e601925e931c15d9482b41e76 Mon Sep 17 00:00:00 2001 From: spisarski Date: Fri, 30 Jul 2021 15:17:44 -0600 Subject: [PATCH 29/40] Example K8s CRDs --- .../kafka-trpt-ddos-detection.yaml | 21 +++++- .../kafka-trpt-drop-clear.yaml | 19 +++++ .../siddhi/kubernetes => examples}/kafka.yaml | 71 +++++++++++-------- examples/test-http.yaml | 36 ++++++++++ examples/test-networking.yaml | 36 ++++++++++ examples/trpt-pv.yaml | 15 ++++ .../trpt-to-kafka.yaml | 25 ++++++- 7 files changed, 191 insertions(+), 32 deletions(-) rename {playbooks/siddhi/kubernetes => examples}/kafka-trpt-ddos-detection.yaml (80%) rename {playbooks/siddhi/kubernetes => examples}/kafka-trpt-drop-clear.yaml (82%) rename {playbooks/siddhi/kubernetes => examples}/kafka.yaml (52%) create mode 100644 examples/test-http.yaml create mode 100644 examples/test-networking.yaml create mode 100644 examples/trpt-pv.yaml rename {playbooks/siddhi/kubernetes => examples}/trpt-to-kafka.yaml (78%) diff --git a/playbooks/siddhi/kubernetes/kafka-trpt-ddos-detection.yaml b/examples/kafka-trpt-ddos-detection.yaml similarity index 80% rename from playbooks/siddhi/kubernetes/kafka-trpt-ddos-detection.yaml rename to examples/kafka-trpt-ddos-detection.yaml index 77f19f68..43cbf4bf 100644 --- a/playbooks/siddhi/kubernetes/kafka-trpt-ddos-detection.yaml +++ b/examples/kafka-trpt-ddos-detection.yaml @@ -11,6 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +--- apiVersion: siddhi.io/v1alpha2 kind: SiddhiProcess metadata: @@ -22,7 +23,7 @@ spec: @source( type="kafka", topic.list="trptPacket", - bootstrap.servers="kafka-service:9092", + bootstrap.servers="10.110.95.121:9092", group.id="test", threading.option="single.thread", @map( @@ -66,3 +67,21 @@ spec: container: image: "spisarski/siddhi" imagePullPolicy: Always + + persistentVolumeClaim: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: standard + volumeMode: Filesystem + + runner: | + state.persistence: + enabled: true + intervalInMin: 5 + revisionsToKeep: 2 + persistenceStore: io.siddhi.distribution.core.persistence.FileSystemPersistenceStore + config: + location: siddhi-app-persistence diff --git a/playbooks/siddhi/kubernetes/kafka-trpt-drop-clear.yaml b/examples/kafka-trpt-drop-clear.yaml similarity index 82% rename from playbooks/siddhi/kubernetes/kafka-trpt-drop-clear.yaml rename to examples/kafka-trpt-drop-clear.yaml index 0be12801..47788873 100644 --- a/playbooks/siddhi/kubernetes/kafka-trpt-drop-clear.yaml +++ b/examples/kafka-trpt-drop-clear.yaml @@ -11,6 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +--- apiVersion: siddhi.io/v1alpha2 kind: SiddhiProcess metadata: @@ -63,3 +64,21 @@ spec: container: image: "spisarski/siddhi" imagePullPolicy: Always + + persistentVolumeClaim: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: standard + volumeMode: Filesystem + + runner: | + state.persistence: + enabled: true + intervalInMin: 5 + revisionsToKeep: 2 + persistenceStore: io.siddhi.distribution.core.persistence.FileSystemPersistenceStore + config: + location: siddhi-app-persistence diff --git a/playbooks/siddhi/kubernetes/kafka.yaml b/examples/kafka.yaml similarity index 52% rename from playbooks/siddhi/kubernetes/kafka.yaml rename to examples/kafka.yaml index ad6b6629..59c6a169 100644 --- a/playbooks/siddhi/kubernetes/kafka.yaml +++ b/examples/kafka.yaml @@ -28,14 +28,17 @@ spec: spec: containers: - name: zoo1 +# image: bitnami/zookeeper image: digitalwonderland/zookeeper +# image: zookeeper +# imagePullPolicy: Always ports: - - containerPort: 2181 + - containerPort: 2181 env: - - name: ZOOKEEPER_ID - value: "1" - - name: ZOOKEEPER_SERVER_1 - value: zoo1 + - name: ZOOKEEPER_ID + value: "1" + - name: ZOOKEEPER_SERVER_1 + value: zoo1 --- apiVersion: v1 @@ -46,15 +49,15 @@ metadata: app: zookeeper-1 spec: ports: - - name: client - port: 2181 - protocol: TCP - - name: follower - port: 2888 - protocol: TCP - - name: leader - port: 3888 - protocol: TCP + - name: client + port: 2181 + protocol: TCP + - name: follower + port: 2888 + protocol: TCP + - name: leader + port: 3888 + protocol: TCP selector: app: zookeeper-1 @@ -67,13 +70,14 @@ metadata: name: kafka spec: ports: - - port: 9092 - name: kafka-port - protocol: TCP + - port: 9092 + targetPort: 9092 + protocol: TCP selector: app: kafka - id: "0" +# id: "0" type: LoadBalancer +# clusterIP: 10.96.1.2 --- kind: Deployment @@ -95,16 +99,25 @@ spec: containers: - name: kafka image: wurstmeister/kafka +# image: bitnami/kafka ports: - - containerPort: 9092 + - containerPort: 9092 env: - - name: KAFKA_ADVERTISED_PORT - value: "9092" - - name: KAFKA_ADVERTISED_HOST_NAME - value: kafka - - name: KAFKA_ZOOKEEPER_CONNECT - value: zoo1:2181 - - name: KAFKA_BROKER_ID - value: "0" - - name: KAFKA_CREATE_TOPICS - value: admintome-test:1:1 + - name: KAFKA_ADVERTISED_PORT + value: "30718" + - name: KAFKA_ADVERTISED_HOST_NAME + value: 10.96.1.2 + - name: KAFKA_ZOOKEEPER_CONNECT + value: zoo1:2181 + - name: KAFKA_BROKER_ID + value: "0" +# - name: KAFKA_CREATE_TOPICS +# value: admintome-test:1:1 +# - name: KAFKA_LISTENERS +# value: LISTENER_PUBLIC://kafka-service:29092,LISTENER_INTERNAL://localhost:9092 +# - name: KAFKA_ADVERTISED_LISTENERS +# value: LISTENER_PUBLIC://kafka-service:29092,LISTENER_INTERNAL://localhost:9092 +# - name: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP +# value: LISTENER_PUBLIC:PLAINTEXT,LISTENER_INTERNAL:PLAINTEXT +# - name: KAFKA_INTER_BROKER_LISTENER_NAME +# value: LISTENER_PUBLIC diff --git a/examples/test-http.yaml b/examples/test-http.yaml new file mode 100644 index 00000000..035f0ecb --- /dev/null +++ b/examples/test-http.yaml @@ -0,0 +1,36 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hello-app +spec: + selector: + matchLabels: + app: hello + replicas: 1 + template: + metadata: + labels: + app: hello + spec: + containers: + - name: hello + image: "gcr.io/google-samples/hello-app:2.0" + +--- +apiVersion: v1 +kind: Service +metadata: + name: ilb-service + annotations: + networking.gke.io/load-balancer-type: "Internal" + labels: + app: hello +spec: + type: LoadBalancer + selector: + app: hello + ports: + - port: 80 + targetPort: 8080 + protocol: TCP diff --git a/examples/test-networking.yaml b/examples/test-networking.yaml new file mode 100644 index 00000000..e7dbb650 --- /dev/null +++ b/examples/test-networking.yaml @@ -0,0 +1,36 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-networking +spec: + selector: + matchLabels: + app: test-networking + replicas: 1 + template: + metadata: + labels: + app: test-networking + spec: + containers: + - name: ubuntu-test + image: praqma/network-multitool + +--- +apiVersion: v1 +kind: Service +metadata: + name: test-networking-service + annotations: + networking.gke.io/load-balancer-type: "Internal" + labels: + app: test-networking +spec: + type: LoadBalancer + selector: + app: test-networking + ports: + - port: 556 + targetPort: 556 + protocol: UDP diff --git a/examples/trpt-pv.yaml b/examples/trpt-pv.yaml new file mode 100644 index 00000000..f21afb6f --- /dev/null +++ b/examples/trpt-pv.yaml @@ -0,0 +1,15 @@ +kind: PersistentVolume +apiVersion: v1 +metadata: + name: siddhi-pv + labels: + type: local +spec: + storageClassName: standard + persistentVolumeReclaimPolicy: Delete + capacity: + storage: 1Gi + accessModes: + - ReadWriteOnce + hostPath: + path: "/home/siddhi_user/" diff --git a/playbooks/siddhi/kubernetes/trpt-to-kafka.yaml b/examples/trpt-to-kafka.yaml similarity index 78% rename from playbooks/siddhi/kubernetes/trpt-to-kafka.yaml rename to examples/trpt-to-kafka.yaml index 1c240575..134a2b95 100644 --- a/playbooks/siddhi/kubernetes/trpt-to-kafka.yaml +++ b/examples/trpt-to-kafka.yaml @@ -11,6 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +--- apiVersion: siddhi.io/v1alpha2 kind: SiddhiProcess metadata: @@ -35,7 +36,7 @@ spec: @sink( type="kafka", topic="trptPacket", - bootstrap.servers="kafka-service:9092", + bootstrap.servers="10.110.95.121:9092", is.binary.message = "false", @map(type="text") ) @@ -44,7 +45,7 @@ spec: @sink( type="kafka", topic="trptDrop", - bootstrap.servers="kafka-service:9092", + bootstrap.servers="10.110.95.121:9092", is.binary.message = "false", @map(type="text") ) @@ -63,3 +64,23 @@ spec: container: image: "spisarski/siddhi" imagePullPolicy: Always + +--- +apiVersion: v1 +kind: Service +metadata: + name: trpt-to-kafka-0 +spec: + type: LoadBalancer + clusterIP: 10.96.100.2 + externalIPs: + - 192.168.86.181 + selector: + siddhi.io/instance: trpt-to-kafka-0 + siddhi.io/name: SiddhiProcess + siddhi.io/part-of: siddhi-operator + siddhi.io/version: 0.2.2 + ports: + - port: 556 + targetPort: 556 + protocol: UDP From f081f9849667bcf90382d97c8cccaa280fcbc730 Mon Sep 17 00:00:00 2001 From: spisarski Date: Mon, 2 Aug 2021 13:10:03 -0600 Subject: [PATCH 30/40] Able to detect attacks but drop packets are not getting to the AE --- automation/p4/tofino/setup.tf | 3 + automation/p4/tofino/variables.tf | 1 + .../scenarios/lab_trial/all-pkt-flood.yml | 27 +++++- playbooks/scenarios/lab_trial/all.yml | 6 +- playbooks/siddhi/docker/Dockerfile | 6 +- .../kafka-trpt-ddos-detection.yaml.j2 | 87 +++++++++++++++++ .../templates/kafka-trpt-drop-clear.yaml.j2 | 84 +++++++++++++++++ .../kubernetes/templates/trpt-pv.yaml.j2 | 15 +++ .../templates/trpt-to-kafka.yaml.j2 | 84 +++++++++++++++++ .../siddhi/templates/minikube_lb.service.j2 | 14 +++ .../templates/minikube_service.service.j2 | 7 +- playbooks/tofino/setup_minikube_siddhi_ae.yml | 94 +++++++++++++++---- playbooks/tofino/setup_nodes-lab_trial.yml | 2 + playbooks/tofino/setup_tofino_switch.yml | 3 +- playbooks/tofino/setup_virt_eth.yml | 5 +- .../topology_template-lab_trial.yaml.j2 | 3 +- playbooks/tofino/tunnel/setup_gre_tunnels.yml | 7 +- 17 files changed, 409 insertions(+), 39 deletions(-) create mode 100644 playbooks/siddhi/kubernetes/templates/kafka-trpt-ddos-detection.yaml.j2 create mode 100644 playbooks/siddhi/kubernetes/templates/kafka-trpt-drop-clear.yaml.j2 create mode 100644 playbooks/siddhi/kubernetes/templates/trpt-pv.yaml.j2 create mode 100644 playbooks/siddhi/kubernetes/templates/trpt-to-kafka.yaml.j2 create mode 100644 playbooks/siddhi/templates/minikube_lb.service.j2 diff --git a/automation/p4/tofino/setup.tf b/automation/p4/tofino/setup.tf index 16d430ec..588a7762 100644 --- a/automation/p4/tofino/setup.tf +++ b/automation/p4/tofino/setup.tf @@ -22,6 +22,8 @@ locals { agg_tun1_ip = var.scenario_name == "lab_trial" ? aws_network_interface.switch_tun_1.1.private_ip: "n/a" agg_tun1_mac = var.scenario_name == "lab_trial" ? aws_network_interface.switch_tun_1.1.mac_address: "n/a" ae_ip = var.scenario_name == "lab_trial" ? aws_instance.ae.private_ip: "n/a" + ae_ip = var.scenario_name == "lab_trial" ? var.ae_k8s_svc_ip: "n/a" + ae_mgmt_ip = var.scenario_name == "lab_trial" ? aws_instance.ae.private_ip: "n/a" ae_tun1_ip = var.scenario_name == "lab_trial" ? aws_network_interface.ae_tun_1.private_ip: "n/a" ae_tun1_mac = var.scenario_name == "lab_trial" ? aws_network_interface.ae_tun_1.mac_address: "n/a" @@ -151,6 +153,7 @@ inet_ip=${local.lab_inet_ip} inet_tun1_ip=${local.lab_inet_tun1_ip} inet_tun1_mac=${local.lab_inet_tun1_mac} ae_ip=${local.ae_ip} +ae_mgmt_ip=${local.ae_mgmt_ip} ae_tun1_ip=${local.ae_tun1_ip} ae_tun1_mac=${local.ae_tun1_mac} switch_user=${var.switch_user} diff --git a/automation/p4/tofino/variables.tf b/automation/p4/tofino/variables.tf index aed573a5..cb0daffc 100644 --- a/automation/p4/tofino/variables.tf +++ b/automation/p4/tofino/variables.tf @@ -66,6 +66,7 @@ variable "p4_grpc_port" {default = "50051"} variable "bf_grpc_port" {default = "50052"} variable "p4_bridge_subnet" {default = "192.168.0.0/24"} variable "sdn_port" {default = "9998"} +variable "ae_k8s_svc_ip" {default = "10.96.0.2"} variable "switch_nic_prfx" {default = "veth"} variable "service_log_level" {default = "DEBUG"} variable "ae_monitor_intf" {default = "core-tun"} diff --git a/playbooks/scenarios/lab_trial/all-pkt-flood.yml b/playbooks/scenarios/lab_trial/all-pkt-flood.yml index 4a852131..90c873cb 100644 --- a/playbooks/scenarios/lab_trial/all-pkt-flood.yml +++ b/playbooks/scenarios/lab_trial/all-pkt-flood.yml @@ -16,16 +16,33 @@ # Start AE service - hosts: ae gather_facts: no - become: yes tasks: - name: Start tps-tofino-ae service + become: yes systemd: name: tps-tofino-ae - state: restarted + state: started - - name: Wait 10 seconds for tps-tofino-ae to fully start - pause: - seconds: 10 + - name: Wait for K8s deployment/trpt-to-kafka-0 to become available + command: kubectl wait --for=condition=available --timeout=600s deployment/trpt-to-kafka-0 -n default + register: trpt_to_kafka_check + retries: 10 + delay: 20 + until: trpt_to_kafka_check is not failed + + - name: Wait for K8s deployment/kafka-trpt-ddos-detect-0 to become available + command: kubectl wait --for=condition=available --timeout=600s deployment/kafka-trpt-ddos-detect-0 -n default + register: ddos_check + retries: 5 + delay: 10 + until: ddos_check is not failed + + - name: Wait for K8s deployment/kafka-trpt-drop-clear-0 to become available + command: kubectl wait --for=condition=available --timeout=600s deployment/kafka-trpt-drop-clear-0 -n default + register: drop_check + retries: 5 + delay: 10 + until: drop_check is not failed # Packet Flood scenarios for UDP and IPv4 - import_playbook: pkt-flood.yml diff --git a/playbooks/scenarios/lab_trial/all.yml b/playbooks/scenarios/lab_trial/all.yml index c9ec0871..677630e4 100644 --- a/playbooks/scenarios/lab_trial/all.yml +++ b/playbooks/scenarios/lab_trial/all.yml @@ -27,11 +27,11 @@ # Data Drop scenarios - import_playbook: all-data-drop.yml +# Switches now operational - Run Packet Flood scenarios to test the AE +- import_playbook: all-pkt-flood.yml + # Drop Reporting scenarios - import_playbook: all-drop-rpt.yml # Packet Performance tests - import_playbook: iperf.yml - -# Switches now operational - Run Packet Flood scenarios to test the AE -- import_playbook: all-pkt-flood.yml diff --git a/playbooks/siddhi/docker/Dockerfile b/playbooks/siddhi/docker/Dockerfile index fd7e412f..57fb1c98 100644 --- a/playbooks/siddhi/docker/Dockerfile +++ b/playbooks/siddhi/docker/Dockerfile @@ -15,11 +15,7 @@ FROM siddhiio/siddhi-runner-alpine:5.1.2 RUN apk --update add net-tools tcpdump sudo RUN apk update && apk add ca-certificates && rm -rf /var/cache/apk/* -#RUN set -ex && apk --no-cache add sudo - -#SHELL ["/bin/bash", "-o", "pipefail", "-c"] -#RUN addgroup siddhi_user sudo -RUN echo 'siddhi_user:siddhi_user' | chpasswd +RUN echo 'siddhi_user:siddhi' | chpasswd RUN echo '%wheel ALL=(ALL) ALL' > /etc/sudoers.d/wheel RUN adduser siddhi_user wheel diff --git a/playbooks/siddhi/kubernetes/templates/kafka-trpt-ddos-detection.yaml.j2 b/playbooks/siddhi/kubernetes/templates/kafka-trpt-ddos-detection.yaml.j2 new file mode 100644 index 00000000..a8dcb0b1 --- /dev/null +++ b/playbooks/siddhi/kubernetes/templates/kafka-trpt-ddos-detection.yaml.j2 @@ -0,0 +1,87 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: siddhi.io/v1alpha2 +kind: SiddhiProcess +metadata: + name: kafka-trpt-ddos-detect +spec: + apps: + - script: | + @App:name("KafkaSourcePacketJSON") + @source( + type="kafka", + topic.list="trptPacket", + bootstrap.servers="{{ kafka_svc_ip }}:9092", + group.id="test", + threading.option="single.thread", + @map( + type="p4-trpt", + @attributes( + src_mac="intHdr.mdStackHdr.origMac", + ip_ver="ipHdr.version", + dst_ip="ipHdr.dstAddr", + dst_port="protoHdr.dstPort" + ) + ) + ) + define stream trptPktStream ( + src_mac string, + ip_ver int, + dst_ip string, + dst_port long); + + @sink( + type="http", + publisher.url="http://{{ sdn_ip }}:9998/aggAttack", + method="POST", + headers="trp:headers", + @map(type="json") + ) + define stream attackStream ( + src_mac string, + ip_ver int, + dst_ip string, + dst_port long, + count long + ); + + @info(name = "trptJsonQuery") + from trptPktStream#window.time(1 sec) + select src_mac, ip_ver, dst_ip, dst_port, count(ip_ver) as count + group by src_mac, dst_ip, dst_port + having count == 10 + insert into attackStream; + + container: + image: "spisarski/siddhi" + imagePullPolicy: Always + + persistentVolumeClaim: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: standard + volumeMode: Filesystem + + runner: | + state.persistence: + enabled: true + intervalInMin: 5 + revisionsToKeep: 2 + persistenceStore: io.siddhi.distribution.core.persistence.FileSystemPersistenceStore + config: + location: siddhi-app-persistence diff --git a/playbooks/siddhi/kubernetes/templates/kafka-trpt-drop-clear.yaml.j2 b/playbooks/siddhi/kubernetes/templates/kafka-trpt-drop-clear.yaml.j2 new file mode 100644 index 00000000..3f80d227 --- /dev/null +++ b/playbooks/siddhi/kubernetes/templates/kafka-trpt-drop-clear.yaml.j2 @@ -0,0 +1,84 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: siddhi.io/v1alpha2 +kind: SiddhiProcess +metadata: + name: kafka-trpt-drop-clear +spec: + apps: + - script: | + @App:name("KafkaSourceDropJSON") + @source( + type="kafka", + topic.list="trptDrop", + bootstrap.servers="{{ kafka_svc_ip }}:9092", + group.id="test", + threading.option="single.thread", + @map( + type="p4-trpt", + @attributes( + timestamp="dropHdr.timestamp", + dropKey="dropHdr.dropKey", + dropCount="dropHdr.dropCount" + ) + ) + ) + define stream trptDropStream ( + timestamp long, + dropKey string, + dropCount long + ); + + @sink( + type="http", + publisher.url="http://{{ sdn_ip }}:9998/aggAttack", + method="DELETE", + headers="trp:headers", + @map(type="json") + ) + define stream dropAttackStream ( + dropKey string, + dropCount long, + count long + ); + + @info(name = "trptJsonQuery") + from trptDropStream#window.time(35 sec) + select dropKey, dropCount, count(dropCount) as count + group by dropKey, dropCount + having count >= 3 + insert into dropAttackStream; + + container: + image: "spisarski/siddhi" + imagePullPolicy: Always + + persistentVolumeClaim: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: standard + volumeMode: Filesystem + + runner: | + state.persistence: + enabled: true + intervalInMin: 5 + revisionsToKeep: 2 + persistenceStore: io.siddhi.distribution.core.persistence.FileSystemPersistenceStore + config: + location: siddhi-app-persistence diff --git a/playbooks/siddhi/kubernetes/templates/trpt-pv.yaml.j2 b/playbooks/siddhi/kubernetes/templates/trpt-pv.yaml.j2 new file mode 100644 index 00000000..f21afb6f --- /dev/null +++ b/playbooks/siddhi/kubernetes/templates/trpt-pv.yaml.j2 @@ -0,0 +1,15 @@ +kind: PersistentVolume +apiVersion: v1 +metadata: + name: siddhi-pv + labels: + type: local +spec: + storageClassName: standard + persistentVolumeReclaimPolicy: Delete + capacity: + storage: 1Gi + accessModes: + - ReadWriteOnce + hostPath: + path: "/home/siddhi_user/" diff --git a/playbooks/siddhi/kubernetes/templates/trpt-to-kafka.yaml.j2 b/playbooks/siddhi/kubernetes/templates/trpt-to-kafka.yaml.j2 new file mode 100644 index 00000000..8bb5a190 --- /dev/null +++ b/playbooks/siddhi/kubernetes/templates/trpt-to-kafka.yaml.j2 @@ -0,0 +1,84 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: siddhi.io/v1alpha2 +kind: SiddhiProcess +metadata: + name: trpt-to-kafka +spec: + apps: + - script: | + @App:name("UDPSourceTRPT") + @source( + type="udp", + listen.port="556", + @map( + type="p4-trpt", + @attributes( + in_type="telemRptHdr.inType", + full_json="jsonString" + ) + ) + ) + define stream trptUdpStream (in_type int, full_json object); + + @sink( + type="kafka", + topic="trptPacket", + bootstrap.servers="{{ kafka_svc_ip }}:9092", + is.binary.message = "false", + @map(type="text") + ) + define stream trptPacket (full_json object); + + @sink( + type="kafka", + topic="trptDrop", + bootstrap.servers="{{ kafka_svc_ip }}:9092", + is.binary.message = "false", + @map(type="text") + ) + define stream trptDrop (full_json object); + + @info(name = "TrptPacket") + from trptUdpStream[in_type != 2] + select full_json + insert into trptPacket; + + @info(name = "TrptDrop") + from trptUdpStream[in_type == 2] + select full_json + insert into trptDrop; + + container: + image: "spisarski/siddhi" + imagePullPolicy: Always + +--- +apiVersion: v1 +kind: Service +metadata: + name: trpt-to-kafka-0 +spec: + type: LoadBalancer + clusterIP: {{ ae_ip }} + selector: + siddhi.io/instance: trpt-to-kafka-0 + siddhi.io/name: SiddhiProcess + siddhi.io/part-of: siddhi-operator + siddhi.io/version: 0.2.2 + ports: + - port: 556 + targetPort: 556 + protocol: UDP diff --git a/playbooks/siddhi/templates/minikube_lb.service.j2 b/playbooks/siddhi/templates/minikube_lb.service.j2 new file mode 100644 index 00000000..bc3ba16d --- /dev/null +++ b/playbooks/siddhi/templates/minikube_lb.service.j2 @@ -0,0 +1,14 @@ +[Unit] +Description=Minikube service for TPS AE +Wants=minikube.service +Requisite={{ minikube_srvc }} +After={{ minikube_srvc }} +PartOf={{ minikube_srvc }} + +[Service] +User=ubuntu +#Group=docker +ExecStart=/usr/bin/minikube tunnel +ExecStop=/usr/bin/systemctl stop minikube.service +[Install] +WantedBy=multi-user.target diff --git a/playbooks/siddhi/templates/minikube_service.service.j2 b/playbooks/siddhi/templates/minikube_service.service.j2 index 17b89ea9..f09fb2d9 100644 --- a/playbooks/siddhi/templates/minikube_service.service.j2 +++ b/playbooks/siddhi/templates/minikube_service.service.j2 @@ -1,14 +1,15 @@ [Unit] Description=Minikube service for TPS AE -Requires=network.target -After=syslog.target network.target +Requires=network.target docker.service +After=syslog.target network.target docker.service +PartOf={{ part_of_srvc }} [Service] Type=oneshot RemainAfterExit=yes User=ubuntu Group=docker -ExecStart=/usr/bin/minikube start +ExecStart=/usr/bin/minikube start --memory=4096 ExecStop=/usr/bin/minikube stop [Install] diff --git a/playbooks/tofino/setup_minikube_siddhi_ae.yml b/playbooks/tofino/setup_minikube_siddhi_ae.yml index f3b35832..881034d6 100644 --- a/playbooks/tofino/setup_minikube_siddhi_ae.yml +++ b/playbooks/tofino/setup_minikube_siddhi_ae.yml @@ -15,13 +15,23 @@ - hosts: "{{ host_val | default('all') }}" gather_facts: no tasks: - - name: Create Service File to /etc/systemd/system/tps-tofino-ae.service + - name: Create Service File to /etc/systemd/system/minikube.service become: yes template: src: ../siddhi/templates/minikube_service.service.j2 + dest: /etc/systemd/system/minikube.service + vars: + part_of_srvc: tps-tofino-ae.service + + - name: Create Service File to /etc/systemd/system/tps-tofino-ae.service + become: yes + template: + src: ../siddhi/templates/minikube_lb.service.j2 dest: /etc/systemd/system/tps-tofino-ae.service + vars: + minikube_srvc: minikube.service - - name: Start tps-tofino-ae.service + - name: Start TPS AE become: yes systemd: name: tps-tofino-ae.service @@ -43,28 +53,80 @@ debug: var: kube_pods_out.stdout_lines - - name: Install siddhi K8s operator - command: "kubectl apply -f {{ item }}" - loop: - - "{{ trans_sec_dir }}/playbooks/siddhi/kubernetes/kafka.yaml" - - "{{ trans_sec_dir }}/playbooks/siddhi/kubernetes/trpt-to-kafka.yaml" - - "{{ trans_sec_dir }}/playbooks/siddhi/kubernetes/kafka-trpt-ddos-detection.yaml" - - "{{ trans_sec_dir }}/playbooks/siddhi/kubernetes/kafka-trpt-drop-clear.yaml" + - name: Ensure {{ remote_scripts_dir }} scripts directory has been created + become: yes + file: + path: "{{ remote_scripts_dir }}" + state: directory + + - name: Query the Kafka Service clusterIP + command: kubectl get svc -n kafka my-cluster-kafka-bootstrap -o jsonpath="{.spec.clusterIPs[0]}" + register: cluster_ip_out - # TODO - Improve validation by for all containing "Running" - - name: Wait for K8s pods to come up... - command: kubectl get po - register: kube_pods_out + - name: Define the K8s CRDs + set_fact: + the_crds: + - "kafka-trpt-ddos-detection.yaml" + - "kafka-trpt-drop-clear.yaml" + - "trpt-to-kafka.yaml" + kafka_svc_ip: "{{ cluster_ip_out.stdout_lines[0] }}" + + - debug: + var: the_crds + + - name: Applying J2 templates of the siddhi K8s CRDs to {{ remote_scripts_dir }} with Kafka service IP - {{ kafka_svc_ip }} + become: yes + template: + src: "{{ trans_sec_dir }}/playbooks/siddhi/kubernetes/templates/{{ item }}.j2" + dest: "{{ remote_scripts_dir }}/{{ item }}" + loop: "{{ the_crds }}" + + - name: Install siddhi K8s SiddhiProcess + command: "kubectl apply -f {{ remote_scripts_dir }}/{{ item }}" + loop: "{{ the_crds }}" + + - name: Wait for K8s deployment/trpt-to-kafka-0 to become available + command: kubectl wait --for=condition=available --timeout=600s deployment/trpt-to-kafka-0 -n default + register: trpt_to_kafka_check retries: 10 + delay: 20 + until: trpt_to_kafka_check is not failed + + - name: Wait for K8s deployment/kafka-trpt-ddos-detect-0 to become available + command: kubectl wait --for=condition=available --timeout=600s deployment/kafka-trpt-ddos-detect-0 -n default + register: ddos_check + retries: 5 + delay: 10 + until: ddos_check is not failed + + - name: Wait for K8s deployment/kafka-trpt-drop-clear-0 to become available + command: kubectl wait --for=condition=available --timeout=600s deployment/kafka-trpt-drop-clear-0 -n default + register: drop_check + retries: 5 delay: 10 - until: kube_pods_out.stdout.find("Running") != -1 and - kube_pods_out.stdout.find("Creating") == -1 and - kube_pods_out.stdout_lines | length >= 9 + until: drop_check is not failed + + - name: Get pods in default namespace + command: kubectl get pods -n default + register: kube_pods_out - name: Show pods debug: var: kube_pods_out.stdout_lines + # TODO - Find a better means to obtain this bridge as things may break as soon as another interface prefaced with "br-" is created + - name: Query for the docker/K8s bridge + shell: "ip -o link show | awk -F': ' '{print $2}'| grep br-" + register: bridge_name_out + + - name: Store the CRDs + set_fact: + docker_bridge_name: "{{ bridge_name_out.stdout_lines[0] }}" + + # TODO - Reconsider this approach which was hacked in place as TRPTs aren't getting routed into the pod + - name: Forward all {{ clone_tun_name }} packets to the Docker/K8s service bridge {{ docker_bridge_name }} + command: "sudo iptables -A FORWARD -i {{ clone_tun_name }} -o {{ docker_bridge_name }} -j ACCEPT" + - name: Stop tps-tofino-ae.service become: yes systemd: diff --git a/playbooks/tofino/setup_nodes-lab_trial.yml b/playbooks/tofino/setup_nodes-lab_trial.yml index 7f343883..539cc9be 100644 --- a/playbooks/tofino/setup_nodes-lab_trial.yml +++ b/playbooks/tofino/setup_nodes-lab_trial.yml @@ -24,6 +24,7 @@ - import_playbook: setup_minikube_siddhi_ae.yml vars: host_val: ae + clone_tun_name: core-tun1 # Create virtual interfaces on switches - import_playbook: setup_virt_eth.yml @@ -51,3 +52,4 @@ port_to_wait: "{{ sdn_port }}" wait_timeout: 60 load_p4: False + diff --git a/playbooks/tofino/setup_tofino_switch.yml b/playbooks/tofino/setup_tofino_switch.yml index 669cf892..efd3dd5d 100644 --- a/playbooks/tofino/setup_tofino_switch.yml +++ b/playbooks/tofino/setup_tofino_switch.yml @@ -27,6 +27,7 @@ # Start tofino model chip emulator - import_playbook: ../general/start_service.yml vars: + setup_for_hw: "{{ from_hw | default(False) }}" host_val: "{{ switches }}" service_name: tps-tofino-model topo_dict: "{{ lookup('file','{{ topo_file_loc }}') | from_yaml }}" @@ -37,7 +38,7 @@ prog_name: "{{ p4_prog }}" additional_tmplt_file: "{{ trans_sec_dir }}/playbooks/tofino/templates/tofino-model-veth-port-mapping.json.j2" additional_tmplt_out_file: "{{ remote_scripts_dir }}/port-mapping.json" - when: not from_hw | bool + when: not setup_for_hw | bool # Start switchd - import_playbook: ../general/start_service.yml diff --git a/playbooks/tofino/setup_virt_eth.yml b/playbooks/tofino/setup_virt_eth.yml index a318d7ab..363428dc 100644 --- a/playbooks/tofino/setup_virt_eth.yml +++ b/playbooks/tofino/setup_virt_eth.yml @@ -20,10 +20,11 @@ SDE: "{{ remote_sde_dir }}" SDE_INSTALL: "{{ remote_sde_dir }}/install" vars: - setup_veth: "{{ not from_hw|bool | default(True) }}" + setup_for_hw: "{{ from_hw | default(False) }}" + setup_veth: "{{ not setup_for_hw | bool }}" tasks: - debug: - var: from_hw + var: setup_for_hw - debug: var: setup_veth - name: Setup DMA diff --git a/playbooks/tofino/templates/topology_template-lab_trial.yaml.j2 b/playbooks/tofino/templates/topology_template-lab_trial.yaml.j2 index a0c356a4..be25796e 100644 --- a/playbooks/tofino/templates/topology_template-lab_trial.yaml.j2 +++ b/playbooks/tofino/templates/topology_template-lab_trial.yaml.j2 @@ -73,7 +73,8 @@ hosts: {% if host_user is defined %} user: {{ host_user }} {% endif %} - public_ip: {{ ae_ip }} + public_ip: {{ ae_mgmt_ip }} + streaming_ip: {{ ae_ip }} tun1_ip: {{ ae_tun1_ip }} tun1_mac: {{ ae_tun1_mac }} tunnels: diff --git a/playbooks/tofino/tunnel/setup_gre_tunnels.yml b/playbooks/tofino/tunnel/setup_gre_tunnels.yml index b1190e6a..e326e644 100644 --- a/playbooks/tofino/tunnel/setup_gre_tunnels.yml +++ b/playbooks/tofino/tunnel/setup_gre_tunnels.yml @@ -19,10 +19,11 @@ gather_facts: yes become: yes vars: - setup_tunnels: "{{ not from_hw|bool | default(True) }}" + setup_for_hw: "{{ from_hw | default(False) }}" + setup_tunnels: "{{ not setup_for_hw | bool }}" tasks: - debug: - var: from_hw + var: setup_for_hw - debug: var: setup_tunnels - block: @@ -38,7 +39,7 @@ name: ip_gre state: present - - name: install ip_gre module + - name: Setup ipv4 forwarding lineinfile: path: /etc/sysctl.conf line: "net.ipv4.ip_forward=1" From a2a072efbe2e0f4b388aa44a96c7b3ddcf82c9cd Mon Sep 17 00:00:00 2001 From: spisarski Date: Wed, 18 Aug 2021 12:56:23 -0600 Subject: [PATCH 31/40] Fix merge issue & create new Siddhi image as upstream custom extensions had minor changes. --- automation/p4/tofino/setup.tf | 1 - automation/p4/tofino/variables.tf | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/automation/p4/tofino/setup.tf b/automation/p4/tofino/setup.tf index 588a7762..e7663cb9 100644 --- a/automation/p4/tofino/setup.tf +++ b/automation/p4/tofino/setup.tf @@ -21,7 +21,6 @@ locals { agg_switch_ip = var.scenario_name == "lab_trial" ? aws_instance.tps-switch.1.private_ip: "n/a" agg_tun1_ip = var.scenario_name == "lab_trial" ? aws_network_interface.switch_tun_1.1.private_ip: "n/a" agg_tun1_mac = var.scenario_name == "lab_trial" ? aws_network_interface.switch_tun_1.1.mac_address: "n/a" - ae_ip = var.scenario_name == "lab_trial" ? aws_instance.ae.private_ip: "n/a" ae_ip = var.scenario_name == "lab_trial" ? var.ae_k8s_svc_ip: "n/a" ae_mgmt_ip = var.scenario_name == "lab_trial" ? aws_instance.ae.private_ip: "n/a" ae_tun1_ip = var.scenario_name == "lab_trial" ? aws_network_interface.ae_tun_1.private_ip: "n/a" diff --git a/automation/p4/tofino/variables.tf b/automation/p4/tofino/variables.tf index cb0daffc..478ad348 100644 --- a/automation/p4/tofino/variables.tf +++ b/automation/p4/tofino/variables.tf @@ -35,7 +35,7 @@ variable "tofino" { } } -variable "siddhi_ae_ami" {default = "ami-07e5e73831d02dbe9"} +variable "siddhi_ae_ami" {default = "ami-010458ceb325ec348"} variable "switch_instance_type" {default = "t2.2xlarge"} variable "ae_instance_type" {default = "t2.2xlarge"} From aad917228c09a3110240f80af65dc803dc021b19 Mon Sep 17 00:00:00 2001 From: spisarski Date: Fri, 23 Jul 2021 15:28:53 -0600 Subject: [PATCH 32/40] Initial changes for converting the TPS AE to run in K8s Created new AE image with minikube and dependencies installed Changed node setup for the lab_trial scenario to load the necessary deployments and siddhi operators for which there are still some issues --- playbooks/env-build/siddhi/env_build.yml | 3 +- playbooks/{siddhi => general}/setup_jdk.yml | 0 playbooks/{siddhi => general}/setup_kafka.yml | 0 playbooks/general/setup_source.yml | 79 +++++++------ playbooks/siddhi/docker/Dockerfile | 16 +++ .../kubernetes/kafka-trpt-ddos-detection.yaml | 68 +++++++++++ .../kubernetes/kafka-trpt-drop-clear.yaml | 65 +++++++++++ playbooks/siddhi/kubernetes/kafka.yaml | 110 ++++++++++++++++++ .../siddhi/kubernetes/trpt-to-kafka.yaml | 65 +++++++++++ playbooks/siddhi/setup_siddhi_maven.yml | 4 +- .../templates/minikube_service.service.j2 | 15 +++ playbooks/tofino/setup_minikube_siddhi_ae.yml | 72 ++++++++++++ playbooks/tofino/setup_nodes-lab_trial.yml | 28 +---- 13 files changed, 461 insertions(+), 64 deletions(-) rename playbooks/{siddhi => general}/setup_jdk.yml (100%) rename playbooks/{siddhi => general}/setup_kafka.yml (100%) create mode 100644 playbooks/siddhi/docker/Dockerfile create mode 100644 playbooks/siddhi/kubernetes/kafka-trpt-ddos-detection.yaml create mode 100644 playbooks/siddhi/kubernetes/kafka-trpt-drop-clear.yaml create mode 100644 playbooks/siddhi/kubernetes/kafka.yaml create mode 100644 playbooks/siddhi/kubernetes/trpt-to-kafka.yaml create mode 100644 playbooks/siddhi/templates/minikube_service.service.j2 create mode 100644 playbooks/tofino/setup_minikube_siddhi_ae.yml diff --git a/playbooks/env-build/siddhi/env_build.yml b/playbooks/env-build/siddhi/env_build.yml index 3ea2f02f..69511097 100644 --- a/playbooks/env-build/siddhi/env_build.yml +++ b/playbooks/env-build/siddhi/env_build.yml @@ -14,4 +14,5 @@ # https://github.com/p4lang/tutorials/blob/master/vm/user-bootstrap.sh # Project and script derived in part from the script in the link above --- -- import_playbook: ../../siddhi/setup_siddhi_maven.yml +#- import_playbook: ../../siddhi/setup_siddhi_maven.yml +- import_playbook: ../../siddhi/setup_siddhi_minikube.yml diff --git a/playbooks/siddhi/setup_jdk.yml b/playbooks/general/setup_jdk.yml similarity index 100% rename from playbooks/siddhi/setup_jdk.yml rename to playbooks/general/setup_jdk.yml diff --git a/playbooks/siddhi/setup_kafka.yml b/playbooks/general/setup_kafka.yml similarity index 100% rename from playbooks/siddhi/setup_kafka.yml rename to playbooks/general/setup_kafka.yml diff --git a/playbooks/general/setup_source.yml b/playbooks/general/setup_source.yml index ae21c9e7..5b5baef4 100644 --- a/playbooks/general/setup_source.yml +++ b/playbooks/general/setup_source.yml @@ -20,48 +20,51 @@ dest_dir: "/home/{{ ansible_user }}/transparent-security" run_tests: "{{ python_unit_tests | default(false) }}" install_python: "{{ install_tps_python | default(true) }}" + install_tps_dependencies: "{{ install_dependencies | default(true) }}" tasks: - - name: Install apt dependencies - apt: - update_cache: yes - name: - - python3-pip - - arping - - iperf3 - register: apt_rc - retries: 3 - delay: 10 - until: apt_rc is not failed - when: ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu' + - block: + - name: Install apt dependencies + apt: + update_cache: yes + name: + - python3-pip + - arping + - iperf3 + register: apt_rc + retries: 3 + delay: 10 + until: apt_rc is not failed + when: ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu' - - name: Install yum dependencies - yum: - update_cache: yes - name: - - python3-pip - - python3-devel - - gcc-c++ - - tcpdump - - net-tools - - iputils - - iperf3 - register: apt_rc - retries: 3 - delay: 10 - until: apt_rc is not failed - when: ansible_distribution == 'CentOS' or ansible_distribution == 'Red Hat Enterprise Linux' + - name: Install yum dependencies + yum: + update_cache: yes + name: + - python3-pip + - python3-devel + - gcc-c++ + - tcpdump + - net-tools + - iputils + - iperf3 + register: apt_rc + retries: 3 + delay: 10 + until: apt_rc is not failed + when: ansible_distribution == 'CentOS' or ansible_distribution == 'Red Hat Enterprise Linux' - - name: Downgrade pip3 scapy version to 2.4.3 due to 2.4.4 bug when running receive_packets.py on centos - pip: - name: - - scapy==2.4.3 - executable: pip3 - register: pip_rc - retries: 3 - delay: 10 - until: pip_rc is not failed - when: ansible_distribution == 'CentOS' or ansible_distribution == 'Red Hat Enterprise Linux' + - name: Downgrade pip3 scapy version to 2.4.3 due to 2.4.4 bug when running receive_packets.py on centos + pip: + name: + - scapy==2.4.3 + executable: pip3 + register: pip_rc + retries: 3 + delay: 10 + until: pip_rc is not failed + when: ansible_distribution == 'CentOS' or ansible_distribution == 'Red Hat Enterprise Linux' + when: install_tps_dependencies | bool - name: Copy local transparent-security source become: no diff --git a/playbooks/siddhi/docker/Dockerfile b/playbooks/siddhi/docker/Dockerfile new file mode 100644 index 00000000..0cf08ce4 --- /dev/null +++ b/playbooks/siddhi/docker/Dockerfile @@ -0,0 +1,16 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM siddhiio/siddhi-runner-alpine:5.1.2 +ARG classpath_lib=. +COPY $classpath_lib/lib/* /home/siddhi_user/siddhi-runner/lib/ diff --git a/playbooks/siddhi/kubernetes/kafka-trpt-ddos-detection.yaml b/playbooks/siddhi/kubernetes/kafka-trpt-ddos-detection.yaml new file mode 100644 index 00000000..77f19f68 --- /dev/null +++ b/playbooks/siddhi/kubernetes/kafka-trpt-ddos-detection.yaml @@ -0,0 +1,68 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: siddhi.io/v1alpha2 +kind: SiddhiProcess +metadata: + name: kafka-trpt-ddos-detect +spec: + apps: + - script: | + @App:name("KafkaSourcePacketJSON") + @source( + type="kafka", + topic.list="trptPacket", + bootstrap.servers="kafka-service:9092", + group.id="test", + threading.option="single.thread", + @map( + type="p4-trpt", + @attributes( + src_mac="intHdr.mdStackHdr.origMac", + ip_ver="ipHdr.version", + dst_ip="ipHdr.dstAddr", + dst_port="protoHdr.dstPort" + ) + ) + ) + define stream trptPktStream ( + src_mac string, + ip_ver int, + dst_ip string, + dst_port long); + + @sink( + type="http", + publisher.url="http://localhost:9998/aggAttack", + method="POST", + headers="trp:headers", + @map(type="json") + ) + define stream attackStream ( + src_mac string, + ip_ver int, + dst_ip string, + dst_port long, + count long + ); + + @info(name = "trptJsonQuery") + from trptPktStream#window.time(1 sec) + select src_mac, ip_ver, dst_ip, dst_port, count(ip_ver) as count + group by src_mac, dst_ip, dst_port + having count == 10 + insert into attackStream; + + container: + image: "spisarski/siddhi" + imagePullPolicy: Always diff --git a/playbooks/siddhi/kubernetes/kafka-trpt-drop-clear.yaml b/playbooks/siddhi/kubernetes/kafka-trpt-drop-clear.yaml new file mode 100644 index 00000000..0be12801 --- /dev/null +++ b/playbooks/siddhi/kubernetes/kafka-trpt-drop-clear.yaml @@ -0,0 +1,65 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: siddhi.io/v1alpha2 +kind: SiddhiProcess +metadata: + name: kafka-trpt-drop-clear +spec: + apps: + - script: | + @App:name("KafkaSourceDropJSON") + @source( + type="kafka", + topic.list="trptDrop", + bootstrap.servers="kafka-service:9092", + group.id="test", + threading.option="single.thread", + @map( + type="p4-trpt", + @attributes( + timestamp="dropHdr.timestamp", + dropKey="dropHdr.dropKey", + dropCount="dropHdr.dropCount" + ) + ) + ) + define stream trptDropStream ( + timestamp long, + dropKey string, + dropCount long + ); + + @sink( + type="http", + publisher.url="http://localhost:9998/aggAttack", + method="DELETE", + headers="trp:headers", + @map(type="json") + ) + define stream dropAttackStream ( + dropKey string, + dropCount long, + count long + ); + + @info(name = "trptJsonQuery") + from trptDropStream#window.time(35 sec) + select dropKey, dropCount, count(dropCount) as count + group by dropKey, dropCount + having count >= 3 + insert into dropAttackStream; + + container: + image: "spisarski/siddhi" + imagePullPolicy: Always diff --git a/playbooks/siddhi/kubernetes/kafka.yaml b/playbooks/siddhi/kubernetes/kafka.yaml new file mode 100644 index 00000000..ad6b6629 --- /dev/null +++ b/playbooks/siddhi/kubernetes/kafka.yaml @@ -0,0 +1,110 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: zookeeper-deploy +spec: + replicas: 2 + selector: + matchLabels: + app: zookeeper-1 + template: + metadata: + labels: + app: zookeeper-1 + spec: + containers: + - name: zoo1 + image: digitalwonderland/zookeeper + ports: + - containerPort: 2181 + env: + - name: ZOOKEEPER_ID + value: "1" + - name: ZOOKEEPER_SERVER_1 + value: zoo1 + +--- +apiVersion: v1 +kind: Service +metadata: + name: zoo1 + labels: + app: zookeeper-1 +spec: + ports: + - name: client + port: 2181 + protocol: TCP + - name: follower + port: 2888 + protocol: TCP + - name: leader + port: 3888 + protocol: TCP + selector: + app: zookeeper-1 + +--- +apiVersion: v1 +kind: Service +metadata: + name: kafka-service + labels: + name: kafka +spec: + ports: + - port: 9092 + name: kafka-port + protocol: TCP + selector: + app: kafka + id: "0" + type: LoadBalancer + +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: kafka-broker0 +spec: + replicas: 2 + selector: + matchLabels: + app: kafka + id: "0" + template: + metadata: + labels: + app: kafka + id: "0" + spec: + containers: + - name: kafka + image: wurstmeister/kafka + ports: + - containerPort: 9092 + env: + - name: KAFKA_ADVERTISED_PORT + value: "9092" + - name: KAFKA_ADVERTISED_HOST_NAME + value: kafka + - name: KAFKA_ZOOKEEPER_CONNECT + value: zoo1:2181 + - name: KAFKA_BROKER_ID + value: "0" + - name: KAFKA_CREATE_TOPICS + value: admintome-test:1:1 diff --git a/playbooks/siddhi/kubernetes/trpt-to-kafka.yaml b/playbooks/siddhi/kubernetes/trpt-to-kafka.yaml new file mode 100644 index 00000000..1c240575 --- /dev/null +++ b/playbooks/siddhi/kubernetes/trpt-to-kafka.yaml @@ -0,0 +1,65 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: siddhi.io/v1alpha2 +kind: SiddhiProcess +metadata: + name: trpt-to-kafka +spec: + apps: + - script: | + @App:name("UDPSourceTRPT") + @source( + type="udp", + listen.port="556", + @map( + type="p4-trpt", + @attributes( + in_type="telemRptHdr.inType", + full_json="jsonString" + ) + ) + ) + define stream trptUdpStream (in_type int, full_json object); + + @sink( + type="kafka", + topic="trptPacket", + bootstrap.servers="kafka-service:9092", + is.binary.message = "false", + @map(type="text") + ) + define stream trptPacket (full_json object); + + @sink( + type="kafka", + topic="trptDrop", + bootstrap.servers="kafka-service:9092", + is.binary.message = "false", + @map(type="text") + ) + define stream trptDrop (full_json object); + + @info(name = "TrptPacket") + from trptUdpStream[in_type != 2] + select full_json + insert into trptPacket; + + @info(name = "TrptDrop") + from trptUdpStream[in_type == 2] + select full_json + insert into trptDrop; + + container: + image: "spisarski/siddhi" + imagePullPolicy: Always diff --git a/playbooks/siddhi/setup_siddhi_maven.yml b/playbooks/siddhi/setup_siddhi_maven.yml index 8be739bc..0871d18d 100644 --- a/playbooks/siddhi/setup_siddhi_maven.yml +++ b/playbooks/siddhi/setup_siddhi_maven.yml @@ -18,8 +18,8 @@ # ctrl-C will gracefully exit. # mvn exec:java -Dexec.mainClass=io.siddhi.extension.map.p4.StartSiddhiRuntime "-Dexec.args=/home/ubuntu/siddhi-map-p4-trpt/docs/siddhi/examples/convert_trpt.siddhi /home/ubuntu/siddhi-map-p4-trpt/docs/siddhi/examples/simple_ddos_detection.siddhi" -f pom.xml --- -- import_playbook: setup_jdk.yml -- import_playbook: setup_kafka.yml +- import_playbook: ../general/setup_jdk.yml +- import_playbook: ../general/setup_kafka.yml - import_playbook: setup_siddhi_p4.yml - import_playbook: ../general/setup_source.yml - import_playbook: ../general/final_env_setup.yml diff --git a/playbooks/siddhi/templates/minikube_service.service.j2 b/playbooks/siddhi/templates/minikube_service.service.j2 new file mode 100644 index 00000000..17b89ea9 --- /dev/null +++ b/playbooks/siddhi/templates/minikube_service.service.j2 @@ -0,0 +1,15 @@ +[Unit] +Description=Minikube service for TPS AE +Requires=network.target +After=syslog.target network.target + +[Service] +Type=oneshot +RemainAfterExit=yes +User=ubuntu +Group=docker +ExecStart=/usr/bin/minikube start +ExecStop=/usr/bin/minikube stop + +[Install] +WantedBy=multi-user.target diff --git a/playbooks/tofino/setup_minikube_siddhi_ae.yml b/playbooks/tofino/setup_minikube_siddhi_ae.yml new file mode 100644 index 00000000..f3b35832 --- /dev/null +++ b/playbooks/tofino/setup_minikube_siddhi_ae.yml @@ -0,0 +1,72 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- hosts: "{{ host_val | default('all') }}" + gather_facts: no + tasks: + - name: Create Service File to /etc/systemd/system/tps-tofino-ae.service + become: yes + template: + src: ../siddhi/templates/minikube_service.service.j2 + dest: /etc/systemd/system/tps-tofino-ae.service + + - name: Start tps-tofino-ae.service + become: yes + systemd: + name: tps-tofino-ae.service + state: restarted + enabled: yes + register: minkube_start_rc + retries: 3 + delay: 5 + until: minkube_start_rc is not failed + + - name: Wait for tps-tofino-ae service to be completely up... + command: kubectl get po + register: kube_pods_out + retries: 10 + delay: 5 + until: kube_pods_out.stdout.find("Running") != -1 + + - name: Show pods + debug: + var: kube_pods_out.stdout_lines + + - name: Install siddhi K8s operator + command: "kubectl apply -f {{ item }}" + loop: + - "{{ trans_sec_dir }}/playbooks/siddhi/kubernetes/kafka.yaml" + - "{{ trans_sec_dir }}/playbooks/siddhi/kubernetes/trpt-to-kafka.yaml" + - "{{ trans_sec_dir }}/playbooks/siddhi/kubernetes/kafka-trpt-ddos-detection.yaml" + - "{{ trans_sec_dir }}/playbooks/siddhi/kubernetes/kafka-trpt-drop-clear.yaml" + + # TODO - Improve validation by for all containing "Running" + - name: Wait for K8s pods to come up... + command: kubectl get po + register: kube_pods_out + retries: 10 + delay: 10 + until: kube_pods_out.stdout.find("Running") != -1 and + kube_pods_out.stdout.find("Creating") == -1 and + kube_pods_out.stdout_lines | length >= 9 + + - name: Show pods + debug: + var: kube_pods_out.stdout_lines + + - name: Stop tps-tofino-ae.service + become: yes + systemd: + name: tps-tofino-ae.service + state: stopped diff --git a/playbooks/tofino/setup_nodes-lab_trial.yml b/playbooks/tofino/setup_nodes-lab_trial.yml index 1216ef11..135c9750 100644 --- a/playbooks/tofino/setup_nodes-lab_trial.yml +++ b/playbooks/tofino/setup_nodes-lab_trial.yml @@ -19,6 +19,11 @@ trans_sec_source_dir: "{{ orch_trans_sec_dir }}" python_unit_tests: false +# Setup AE +- import_playbook: setup_minikube_siddhi_ae.yml + vars: + host_val: ae + # Create virtual interfaces on switches - import_playbook: setup_virt_eth.yml vars: @@ -45,26 +50,3 @@ port_to_wait: "{{ sdn_port }}" wait_timeout: 60 load_p4: False - -# Start AE -- import_playbook: ../general/start_service.yml - vars: - host_val: ae - service_name: tps-tofino-ae - srvc_start_pause_time: 45 - local_srvc_script_tmplt_file: "{{ orch_trans_sec_dir }}/playbooks/general/templates/siddhi_p4_service.sh.j2" - srvc_type: SIMPLE - sdn_url: "http://{{ sdn_ip }}:{{ sdn_port }}" - telem_rpt_port: 556 - kafka_host_port: localhost:9092 - kafka_trpt_pkt_topic: trptPacket - kafka_trpt_drop_topic: trptDrop - alert_pkt_count: 10 - alert_window_secs: 1 - templates: - - src: "{{ orch_trans_sec_dir }}/playbooks/siddhi/templates/convert_trpt.siddhi.j2" - dest: "{{ remote_scripts_dir }}/convert_trpt.siddhi" - - src: "{{ orch_trans_sec_dir }}/playbooks/siddhi/templates/simple_ddos_detection.siddhi.j2" - dest: "{{ remote_scripts_dir }}/simple_ddos_detection.siddhi" - - src: "{{ orch_trans_sec_dir }}/playbooks/siddhi/templates/simple_ddos_clear_drop.siddhi.j2" - dest: "{{ remote_scripts_dir }}/simple_ddos_clear.siddhi" From fa44249eded837f49683720f985bee48d773763d Mon Sep 17 00:00:00 2001 From: spisarski Date: Wed, 28 Jul 2021 15:05:22 -0600 Subject: [PATCH 33/40] added utilities and sudo password for debugging to custom siddhi runner image --- playbooks/siddhi/docker/Dockerfile | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/playbooks/siddhi/docker/Dockerfile b/playbooks/siddhi/docker/Dockerfile index 0cf08ce4..fd7e412f 100644 --- a/playbooks/siddhi/docker/Dockerfile +++ b/playbooks/siddhi/docker/Dockerfile @@ -12,5 +12,16 @@ # limitations under the License. FROM siddhiio/siddhi-runner-alpine:5.1.2 + +RUN apk --update add net-tools tcpdump sudo +RUN apk update && apk add ca-certificates && rm -rf /var/cache/apk/* +#RUN set -ex && apk --no-cache add sudo + +#SHELL ["/bin/bash", "-o", "pipefail", "-c"] +#RUN addgroup siddhi_user sudo +RUN echo 'siddhi_user:siddhi_user' | chpasswd +RUN echo '%wheel ALL=(ALL) ALL' > /etc/sudoers.d/wheel +RUN adduser siddhi_user wheel + ARG classpath_lib=. COPY $classpath_lib/lib/* /home/siddhi_user/siddhi-runner/lib/ From 83a8387515438a3532b881241e55fa7e1b56ef91 Mon Sep 17 00:00:00 2001 From: spisarski Date: Wed, 28 Jul 2021 15:11:17 -0600 Subject: [PATCH 34/40] housekeeping --- playbooks/tofino/setup_nodes-lab_trial.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/playbooks/tofino/setup_nodes-lab_trial.yml b/playbooks/tofino/setup_nodes-lab_trial.yml index 135c9750..7f343883 100644 --- a/playbooks/tofino/setup_nodes-lab_trial.yml +++ b/playbooks/tofino/setup_nodes-lab_trial.yml @@ -11,6 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +--- # Copy TPS source from orchestrator to all hosts - import_playbook: ../general/setup_source.yml vars: From 0b9764407e947a08db2397cf8ee24d2e53ba0e11 Mon Sep 17 00:00:00 2001 From: spisarski Date: Fri, 30 Jul 2021 15:17:44 -0600 Subject: [PATCH 35/40] Example K8s CRDs --- .../kafka-trpt-ddos-detection.yaml | 21 +++++- .../kafka-trpt-drop-clear.yaml | 19 +++++ .../siddhi/kubernetes => examples}/kafka.yaml | 71 +++++++++++-------- examples/test-http.yaml | 36 ++++++++++ examples/test-networking.yaml | 36 ++++++++++ examples/trpt-pv.yaml | 15 ++++ .../trpt-to-kafka.yaml | 25 ++++++- 7 files changed, 191 insertions(+), 32 deletions(-) rename {playbooks/siddhi/kubernetes => examples}/kafka-trpt-ddos-detection.yaml (80%) rename {playbooks/siddhi/kubernetes => examples}/kafka-trpt-drop-clear.yaml (82%) rename {playbooks/siddhi/kubernetes => examples}/kafka.yaml (52%) create mode 100644 examples/test-http.yaml create mode 100644 examples/test-networking.yaml create mode 100644 examples/trpt-pv.yaml rename {playbooks/siddhi/kubernetes => examples}/trpt-to-kafka.yaml (78%) diff --git a/playbooks/siddhi/kubernetes/kafka-trpt-ddos-detection.yaml b/examples/kafka-trpt-ddos-detection.yaml similarity index 80% rename from playbooks/siddhi/kubernetes/kafka-trpt-ddos-detection.yaml rename to examples/kafka-trpt-ddos-detection.yaml index 77f19f68..43cbf4bf 100644 --- a/playbooks/siddhi/kubernetes/kafka-trpt-ddos-detection.yaml +++ b/examples/kafka-trpt-ddos-detection.yaml @@ -11,6 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +--- apiVersion: siddhi.io/v1alpha2 kind: SiddhiProcess metadata: @@ -22,7 +23,7 @@ spec: @source( type="kafka", topic.list="trptPacket", - bootstrap.servers="kafka-service:9092", + bootstrap.servers="10.110.95.121:9092", group.id="test", threading.option="single.thread", @map( @@ -66,3 +67,21 @@ spec: container: image: "spisarski/siddhi" imagePullPolicy: Always + + persistentVolumeClaim: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: standard + volumeMode: Filesystem + + runner: | + state.persistence: + enabled: true + intervalInMin: 5 + revisionsToKeep: 2 + persistenceStore: io.siddhi.distribution.core.persistence.FileSystemPersistenceStore + config: + location: siddhi-app-persistence diff --git a/playbooks/siddhi/kubernetes/kafka-trpt-drop-clear.yaml b/examples/kafka-trpt-drop-clear.yaml similarity index 82% rename from playbooks/siddhi/kubernetes/kafka-trpt-drop-clear.yaml rename to examples/kafka-trpt-drop-clear.yaml index 0be12801..47788873 100644 --- a/playbooks/siddhi/kubernetes/kafka-trpt-drop-clear.yaml +++ b/examples/kafka-trpt-drop-clear.yaml @@ -11,6 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +--- apiVersion: siddhi.io/v1alpha2 kind: SiddhiProcess metadata: @@ -63,3 +64,21 @@ spec: container: image: "spisarski/siddhi" imagePullPolicy: Always + + persistentVolumeClaim: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: standard + volumeMode: Filesystem + + runner: | + state.persistence: + enabled: true + intervalInMin: 5 + revisionsToKeep: 2 + persistenceStore: io.siddhi.distribution.core.persistence.FileSystemPersistenceStore + config: + location: siddhi-app-persistence diff --git a/playbooks/siddhi/kubernetes/kafka.yaml b/examples/kafka.yaml similarity index 52% rename from playbooks/siddhi/kubernetes/kafka.yaml rename to examples/kafka.yaml index ad6b6629..59c6a169 100644 --- a/playbooks/siddhi/kubernetes/kafka.yaml +++ b/examples/kafka.yaml @@ -28,14 +28,17 @@ spec: spec: containers: - name: zoo1 +# image: bitnami/zookeeper image: digitalwonderland/zookeeper +# image: zookeeper +# imagePullPolicy: Always ports: - - containerPort: 2181 + - containerPort: 2181 env: - - name: ZOOKEEPER_ID - value: "1" - - name: ZOOKEEPER_SERVER_1 - value: zoo1 + - name: ZOOKEEPER_ID + value: "1" + - name: ZOOKEEPER_SERVER_1 + value: zoo1 --- apiVersion: v1 @@ -46,15 +49,15 @@ metadata: app: zookeeper-1 spec: ports: - - name: client - port: 2181 - protocol: TCP - - name: follower - port: 2888 - protocol: TCP - - name: leader - port: 3888 - protocol: TCP + - name: client + port: 2181 + protocol: TCP + - name: follower + port: 2888 + protocol: TCP + - name: leader + port: 3888 + protocol: TCP selector: app: zookeeper-1 @@ -67,13 +70,14 @@ metadata: name: kafka spec: ports: - - port: 9092 - name: kafka-port - protocol: TCP + - port: 9092 + targetPort: 9092 + protocol: TCP selector: app: kafka - id: "0" +# id: "0" type: LoadBalancer +# clusterIP: 10.96.1.2 --- kind: Deployment @@ -95,16 +99,25 @@ spec: containers: - name: kafka image: wurstmeister/kafka +# image: bitnami/kafka ports: - - containerPort: 9092 + - containerPort: 9092 env: - - name: KAFKA_ADVERTISED_PORT - value: "9092" - - name: KAFKA_ADVERTISED_HOST_NAME - value: kafka - - name: KAFKA_ZOOKEEPER_CONNECT - value: zoo1:2181 - - name: KAFKA_BROKER_ID - value: "0" - - name: KAFKA_CREATE_TOPICS - value: admintome-test:1:1 + - name: KAFKA_ADVERTISED_PORT + value: "30718" + - name: KAFKA_ADVERTISED_HOST_NAME + value: 10.96.1.2 + - name: KAFKA_ZOOKEEPER_CONNECT + value: zoo1:2181 + - name: KAFKA_BROKER_ID + value: "0" +# - name: KAFKA_CREATE_TOPICS +# value: admintome-test:1:1 +# - name: KAFKA_LISTENERS +# value: LISTENER_PUBLIC://kafka-service:29092,LISTENER_INTERNAL://localhost:9092 +# - name: KAFKA_ADVERTISED_LISTENERS +# value: LISTENER_PUBLIC://kafka-service:29092,LISTENER_INTERNAL://localhost:9092 +# - name: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP +# value: LISTENER_PUBLIC:PLAINTEXT,LISTENER_INTERNAL:PLAINTEXT +# - name: KAFKA_INTER_BROKER_LISTENER_NAME +# value: LISTENER_PUBLIC diff --git a/examples/test-http.yaml b/examples/test-http.yaml new file mode 100644 index 00000000..035f0ecb --- /dev/null +++ b/examples/test-http.yaml @@ -0,0 +1,36 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hello-app +spec: + selector: + matchLabels: + app: hello + replicas: 1 + template: + metadata: + labels: + app: hello + spec: + containers: + - name: hello + image: "gcr.io/google-samples/hello-app:2.0" + +--- +apiVersion: v1 +kind: Service +metadata: + name: ilb-service + annotations: + networking.gke.io/load-balancer-type: "Internal" + labels: + app: hello +spec: + type: LoadBalancer + selector: + app: hello + ports: + - port: 80 + targetPort: 8080 + protocol: TCP diff --git a/examples/test-networking.yaml b/examples/test-networking.yaml new file mode 100644 index 00000000..e7dbb650 --- /dev/null +++ b/examples/test-networking.yaml @@ -0,0 +1,36 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-networking +spec: + selector: + matchLabels: + app: test-networking + replicas: 1 + template: + metadata: + labels: + app: test-networking + spec: + containers: + - name: ubuntu-test + image: praqma/network-multitool + +--- +apiVersion: v1 +kind: Service +metadata: + name: test-networking-service + annotations: + networking.gke.io/load-balancer-type: "Internal" + labels: + app: test-networking +spec: + type: LoadBalancer + selector: + app: test-networking + ports: + - port: 556 + targetPort: 556 + protocol: UDP diff --git a/examples/trpt-pv.yaml b/examples/trpt-pv.yaml new file mode 100644 index 00000000..f21afb6f --- /dev/null +++ b/examples/trpt-pv.yaml @@ -0,0 +1,15 @@ +kind: PersistentVolume +apiVersion: v1 +metadata: + name: siddhi-pv + labels: + type: local +spec: + storageClassName: standard + persistentVolumeReclaimPolicy: Delete + capacity: + storage: 1Gi + accessModes: + - ReadWriteOnce + hostPath: + path: "/home/siddhi_user/" diff --git a/playbooks/siddhi/kubernetes/trpt-to-kafka.yaml b/examples/trpt-to-kafka.yaml similarity index 78% rename from playbooks/siddhi/kubernetes/trpt-to-kafka.yaml rename to examples/trpt-to-kafka.yaml index 1c240575..134a2b95 100644 --- a/playbooks/siddhi/kubernetes/trpt-to-kafka.yaml +++ b/examples/trpt-to-kafka.yaml @@ -11,6 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +--- apiVersion: siddhi.io/v1alpha2 kind: SiddhiProcess metadata: @@ -35,7 +36,7 @@ spec: @sink( type="kafka", topic="trptPacket", - bootstrap.servers="kafka-service:9092", + bootstrap.servers="10.110.95.121:9092", is.binary.message = "false", @map(type="text") ) @@ -44,7 +45,7 @@ spec: @sink( type="kafka", topic="trptDrop", - bootstrap.servers="kafka-service:9092", + bootstrap.servers="10.110.95.121:9092", is.binary.message = "false", @map(type="text") ) @@ -63,3 +64,23 @@ spec: container: image: "spisarski/siddhi" imagePullPolicy: Always + +--- +apiVersion: v1 +kind: Service +metadata: + name: trpt-to-kafka-0 +spec: + type: LoadBalancer + clusterIP: 10.96.100.2 + externalIPs: + - 192.168.86.181 + selector: + siddhi.io/instance: trpt-to-kafka-0 + siddhi.io/name: SiddhiProcess + siddhi.io/part-of: siddhi-operator + siddhi.io/version: 0.2.2 + ports: + - port: 556 + targetPort: 556 + protocol: UDP From 6d8c29ec758e9c6f575db993d85be31f8d0cccb0 Mon Sep 17 00:00:00 2001 From: spisarski Date: Mon, 2 Aug 2021 13:10:03 -0600 Subject: [PATCH 36/40] Able to detect attacks but drop packets are not getting to the AE --- automation/p4/tofino/setup.tf | 3 + automation/p4/tofino/variables.tf | 1 + .../scenarios/lab_trial/all-pkt-flood.yml | 27 +++++- playbooks/scenarios/lab_trial/all.yml | 6 +- playbooks/siddhi/docker/Dockerfile | 6 +- .../kafka-trpt-ddos-detection.yaml.j2 | 87 +++++++++++++++++ .../templates/kafka-trpt-drop-clear.yaml.j2 | 84 +++++++++++++++++ .../kubernetes/templates/trpt-pv.yaml.j2 | 15 +++ .../templates/trpt-to-kafka.yaml.j2 | 84 +++++++++++++++++ .../siddhi/templates/minikube_lb.service.j2 | 14 +++ .../templates/minikube_service.service.j2 | 7 +- playbooks/tofino/setup_minikube_siddhi_ae.yml | 94 +++++++++++++++---- playbooks/tofino/setup_nodes-lab_trial.yml | 2 + playbooks/tofino/setup_tofino_switch.yml | 3 +- playbooks/tofino/setup_virt_eth.yml | 5 +- .../topology_template-lab_trial.yaml.j2 | 3 +- playbooks/tofino/tunnel/setup_gre_tunnels.yml | 7 +- 17 files changed, 409 insertions(+), 39 deletions(-) create mode 100644 playbooks/siddhi/kubernetes/templates/kafka-trpt-ddos-detection.yaml.j2 create mode 100644 playbooks/siddhi/kubernetes/templates/kafka-trpt-drop-clear.yaml.j2 create mode 100644 playbooks/siddhi/kubernetes/templates/trpt-pv.yaml.j2 create mode 100644 playbooks/siddhi/kubernetes/templates/trpt-to-kafka.yaml.j2 create mode 100644 playbooks/siddhi/templates/minikube_lb.service.j2 diff --git a/automation/p4/tofino/setup.tf b/automation/p4/tofino/setup.tf index 16d430ec..588a7762 100644 --- a/automation/p4/tofino/setup.tf +++ b/automation/p4/tofino/setup.tf @@ -22,6 +22,8 @@ locals { agg_tun1_ip = var.scenario_name == "lab_trial" ? aws_network_interface.switch_tun_1.1.private_ip: "n/a" agg_tun1_mac = var.scenario_name == "lab_trial" ? aws_network_interface.switch_tun_1.1.mac_address: "n/a" ae_ip = var.scenario_name == "lab_trial" ? aws_instance.ae.private_ip: "n/a" + ae_ip = var.scenario_name == "lab_trial" ? var.ae_k8s_svc_ip: "n/a" + ae_mgmt_ip = var.scenario_name == "lab_trial" ? aws_instance.ae.private_ip: "n/a" ae_tun1_ip = var.scenario_name == "lab_trial" ? aws_network_interface.ae_tun_1.private_ip: "n/a" ae_tun1_mac = var.scenario_name == "lab_trial" ? aws_network_interface.ae_tun_1.mac_address: "n/a" @@ -151,6 +153,7 @@ inet_ip=${local.lab_inet_ip} inet_tun1_ip=${local.lab_inet_tun1_ip} inet_tun1_mac=${local.lab_inet_tun1_mac} ae_ip=${local.ae_ip} +ae_mgmt_ip=${local.ae_mgmt_ip} ae_tun1_ip=${local.ae_tun1_ip} ae_tun1_mac=${local.ae_tun1_mac} switch_user=${var.switch_user} diff --git a/automation/p4/tofino/variables.tf b/automation/p4/tofino/variables.tf index aed573a5..cb0daffc 100644 --- a/automation/p4/tofino/variables.tf +++ b/automation/p4/tofino/variables.tf @@ -66,6 +66,7 @@ variable "p4_grpc_port" {default = "50051"} variable "bf_grpc_port" {default = "50052"} variable "p4_bridge_subnet" {default = "192.168.0.0/24"} variable "sdn_port" {default = "9998"} +variable "ae_k8s_svc_ip" {default = "10.96.0.2"} variable "switch_nic_prfx" {default = "veth"} variable "service_log_level" {default = "DEBUG"} variable "ae_monitor_intf" {default = "core-tun"} diff --git a/playbooks/scenarios/lab_trial/all-pkt-flood.yml b/playbooks/scenarios/lab_trial/all-pkt-flood.yml index 4a852131..90c873cb 100644 --- a/playbooks/scenarios/lab_trial/all-pkt-flood.yml +++ b/playbooks/scenarios/lab_trial/all-pkt-flood.yml @@ -16,16 +16,33 @@ # Start AE service - hosts: ae gather_facts: no - become: yes tasks: - name: Start tps-tofino-ae service + become: yes systemd: name: tps-tofino-ae - state: restarted + state: started - - name: Wait 10 seconds for tps-tofino-ae to fully start - pause: - seconds: 10 + - name: Wait for K8s deployment/trpt-to-kafka-0 to become available + command: kubectl wait --for=condition=available --timeout=600s deployment/trpt-to-kafka-0 -n default + register: trpt_to_kafka_check + retries: 10 + delay: 20 + until: trpt_to_kafka_check is not failed + + - name: Wait for K8s deployment/kafka-trpt-ddos-detect-0 to become available + command: kubectl wait --for=condition=available --timeout=600s deployment/kafka-trpt-ddos-detect-0 -n default + register: ddos_check + retries: 5 + delay: 10 + until: ddos_check is not failed + + - name: Wait for K8s deployment/kafka-trpt-drop-clear-0 to become available + command: kubectl wait --for=condition=available --timeout=600s deployment/kafka-trpt-drop-clear-0 -n default + register: drop_check + retries: 5 + delay: 10 + until: drop_check is not failed # Packet Flood scenarios for UDP and IPv4 - import_playbook: pkt-flood.yml diff --git a/playbooks/scenarios/lab_trial/all.yml b/playbooks/scenarios/lab_trial/all.yml index c9ec0871..677630e4 100644 --- a/playbooks/scenarios/lab_trial/all.yml +++ b/playbooks/scenarios/lab_trial/all.yml @@ -27,11 +27,11 @@ # Data Drop scenarios - import_playbook: all-data-drop.yml +# Switches now operational - Run Packet Flood scenarios to test the AE +- import_playbook: all-pkt-flood.yml + # Drop Reporting scenarios - import_playbook: all-drop-rpt.yml # Packet Performance tests - import_playbook: iperf.yml - -# Switches now operational - Run Packet Flood scenarios to test the AE -- import_playbook: all-pkt-flood.yml diff --git a/playbooks/siddhi/docker/Dockerfile b/playbooks/siddhi/docker/Dockerfile index fd7e412f..57fb1c98 100644 --- a/playbooks/siddhi/docker/Dockerfile +++ b/playbooks/siddhi/docker/Dockerfile @@ -15,11 +15,7 @@ FROM siddhiio/siddhi-runner-alpine:5.1.2 RUN apk --update add net-tools tcpdump sudo RUN apk update && apk add ca-certificates && rm -rf /var/cache/apk/* -#RUN set -ex && apk --no-cache add sudo - -#SHELL ["/bin/bash", "-o", "pipefail", "-c"] -#RUN addgroup siddhi_user sudo -RUN echo 'siddhi_user:siddhi_user' | chpasswd +RUN echo 'siddhi_user:siddhi' | chpasswd RUN echo '%wheel ALL=(ALL) ALL' > /etc/sudoers.d/wheel RUN adduser siddhi_user wheel diff --git a/playbooks/siddhi/kubernetes/templates/kafka-trpt-ddos-detection.yaml.j2 b/playbooks/siddhi/kubernetes/templates/kafka-trpt-ddos-detection.yaml.j2 new file mode 100644 index 00000000..a8dcb0b1 --- /dev/null +++ b/playbooks/siddhi/kubernetes/templates/kafka-trpt-ddos-detection.yaml.j2 @@ -0,0 +1,87 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: siddhi.io/v1alpha2 +kind: SiddhiProcess +metadata: + name: kafka-trpt-ddos-detect +spec: + apps: + - script: | + @App:name("KafkaSourcePacketJSON") + @source( + type="kafka", + topic.list="trptPacket", + bootstrap.servers="{{ kafka_svc_ip }}:9092", + group.id="test", + threading.option="single.thread", + @map( + type="p4-trpt", + @attributes( + src_mac="intHdr.mdStackHdr.origMac", + ip_ver="ipHdr.version", + dst_ip="ipHdr.dstAddr", + dst_port="protoHdr.dstPort" + ) + ) + ) + define stream trptPktStream ( + src_mac string, + ip_ver int, + dst_ip string, + dst_port long); + + @sink( + type="http", + publisher.url="http://{{ sdn_ip }}:9998/aggAttack", + method="POST", + headers="trp:headers", + @map(type="json") + ) + define stream attackStream ( + src_mac string, + ip_ver int, + dst_ip string, + dst_port long, + count long + ); + + @info(name = "trptJsonQuery") + from trptPktStream#window.time(1 sec) + select src_mac, ip_ver, dst_ip, dst_port, count(ip_ver) as count + group by src_mac, dst_ip, dst_port + having count == 10 + insert into attackStream; + + container: + image: "spisarski/siddhi" + imagePullPolicy: Always + + persistentVolumeClaim: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: standard + volumeMode: Filesystem + + runner: | + state.persistence: + enabled: true + intervalInMin: 5 + revisionsToKeep: 2 + persistenceStore: io.siddhi.distribution.core.persistence.FileSystemPersistenceStore + config: + location: siddhi-app-persistence diff --git a/playbooks/siddhi/kubernetes/templates/kafka-trpt-drop-clear.yaml.j2 b/playbooks/siddhi/kubernetes/templates/kafka-trpt-drop-clear.yaml.j2 new file mode 100644 index 00000000..3f80d227 --- /dev/null +++ b/playbooks/siddhi/kubernetes/templates/kafka-trpt-drop-clear.yaml.j2 @@ -0,0 +1,84 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: siddhi.io/v1alpha2 +kind: SiddhiProcess +metadata: + name: kafka-trpt-drop-clear +spec: + apps: + - script: | + @App:name("KafkaSourceDropJSON") + @source( + type="kafka", + topic.list="trptDrop", + bootstrap.servers="{{ kafka_svc_ip }}:9092", + group.id="test", + threading.option="single.thread", + @map( + type="p4-trpt", + @attributes( + timestamp="dropHdr.timestamp", + dropKey="dropHdr.dropKey", + dropCount="dropHdr.dropCount" + ) + ) + ) + define stream trptDropStream ( + timestamp long, + dropKey string, + dropCount long + ); + + @sink( + type="http", + publisher.url="http://{{ sdn_ip }}:9998/aggAttack", + method="DELETE", + headers="trp:headers", + @map(type="json") + ) + define stream dropAttackStream ( + dropKey string, + dropCount long, + count long + ); + + @info(name = "trptJsonQuery") + from trptDropStream#window.time(35 sec) + select dropKey, dropCount, count(dropCount) as count + group by dropKey, dropCount + having count >= 3 + insert into dropAttackStream; + + container: + image: "spisarski/siddhi" + imagePullPolicy: Always + + persistentVolumeClaim: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: standard + volumeMode: Filesystem + + runner: | + state.persistence: + enabled: true + intervalInMin: 5 + revisionsToKeep: 2 + persistenceStore: io.siddhi.distribution.core.persistence.FileSystemPersistenceStore + config: + location: siddhi-app-persistence diff --git a/playbooks/siddhi/kubernetes/templates/trpt-pv.yaml.j2 b/playbooks/siddhi/kubernetes/templates/trpt-pv.yaml.j2 new file mode 100644 index 00000000..f21afb6f --- /dev/null +++ b/playbooks/siddhi/kubernetes/templates/trpt-pv.yaml.j2 @@ -0,0 +1,15 @@ +kind: PersistentVolume +apiVersion: v1 +metadata: + name: siddhi-pv + labels: + type: local +spec: + storageClassName: standard + persistentVolumeReclaimPolicy: Delete + capacity: + storage: 1Gi + accessModes: + - ReadWriteOnce + hostPath: + path: "/home/siddhi_user/" diff --git a/playbooks/siddhi/kubernetes/templates/trpt-to-kafka.yaml.j2 b/playbooks/siddhi/kubernetes/templates/trpt-to-kafka.yaml.j2 new file mode 100644 index 00000000..8bb5a190 --- /dev/null +++ b/playbooks/siddhi/kubernetes/templates/trpt-to-kafka.yaml.j2 @@ -0,0 +1,84 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: siddhi.io/v1alpha2 +kind: SiddhiProcess +metadata: + name: trpt-to-kafka +spec: + apps: + - script: | + @App:name("UDPSourceTRPT") + @source( + type="udp", + listen.port="556", + @map( + type="p4-trpt", + @attributes( + in_type="telemRptHdr.inType", + full_json="jsonString" + ) + ) + ) + define stream trptUdpStream (in_type int, full_json object); + + @sink( + type="kafka", + topic="trptPacket", + bootstrap.servers="{{ kafka_svc_ip }}:9092", + is.binary.message = "false", + @map(type="text") + ) + define stream trptPacket (full_json object); + + @sink( + type="kafka", + topic="trptDrop", + bootstrap.servers="{{ kafka_svc_ip }}:9092", + is.binary.message = "false", + @map(type="text") + ) + define stream trptDrop (full_json object); + + @info(name = "TrptPacket") + from trptUdpStream[in_type != 2] + select full_json + insert into trptPacket; + + @info(name = "TrptDrop") + from trptUdpStream[in_type == 2] + select full_json + insert into trptDrop; + + container: + image: "spisarski/siddhi" + imagePullPolicy: Always + +--- +apiVersion: v1 +kind: Service +metadata: + name: trpt-to-kafka-0 +spec: + type: LoadBalancer + clusterIP: {{ ae_ip }} + selector: + siddhi.io/instance: trpt-to-kafka-0 + siddhi.io/name: SiddhiProcess + siddhi.io/part-of: siddhi-operator + siddhi.io/version: 0.2.2 + ports: + - port: 556 + targetPort: 556 + protocol: UDP diff --git a/playbooks/siddhi/templates/minikube_lb.service.j2 b/playbooks/siddhi/templates/minikube_lb.service.j2 new file mode 100644 index 00000000..bc3ba16d --- /dev/null +++ b/playbooks/siddhi/templates/minikube_lb.service.j2 @@ -0,0 +1,14 @@ +[Unit] +Description=Minikube service for TPS AE +Wants=minikube.service +Requisite={{ minikube_srvc }} +After={{ minikube_srvc }} +PartOf={{ minikube_srvc }} + +[Service] +User=ubuntu +#Group=docker +ExecStart=/usr/bin/minikube tunnel +ExecStop=/usr/bin/systemctl stop minikube.service +[Install] +WantedBy=multi-user.target diff --git a/playbooks/siddhi/templates/minikube_service.service.j2 b/playbooks/siddhi/templates/minikube_service.service.j2 index 17b89ea9..f09fb2d9 100644 --- a/playbooks/siddhi/templates/minikube_service.service.j2 +++ b/playbooks/siddhi/templates/minikube_service.service.j2 @@ -1,14 +1,15 @@ [Unit] Description=Minikube service for TPS AE -Requires=network.target -After=syslog.target network.target +Requires=network.target docker.service +After=syslog.target network.target docker.service +PartOf={{ part_of_srvc }} [Service] Type=oneshot RemainAfterExit=yes User=ubuntu Group=docker -ExecStart=/usr/bin/minikube start +ExecStart=/usr/bin/minikube start --memory=4096 ExecStop=/usr/bin/minikube stop [Install] diff --git a/playbooks/tofino/setup_minikube_siddhi_ae.yml b/playbooks/tofino/setup_minikube_siddhi_ae.yml index f3b35832..881034d6 100644 --- a/playbooks/tofino/setup_minikube_siddhi_ae.yml +++ b/playbooks/tofino/setup_minikube_siddhi_ae.yml @@ -15,13 +15,23 @@ - hosts: "{{ host_val | default('all') }}" gather_facts: no tasks: - - name: Create Service File to /etc/systemd/system/tps-tofino-ae.service + - name: Create Service File to /etc/systemd/system/minikube.service become: yes template: src: ../siddhi/templates/minikube_service.service.j2 + dest: /etc/systemd/system/minikube.service + vars: + part_of_srvc: tps-tofino-ae.service + + - name: Create Service File to /etc/systemd/system/tps-tofino-ae.service + become: yes + template: + src: ../siddhi/templates/minikube_lb.service.j2 dest: /etc/systemd/system/tps-tofino-ae.service + vars: + minikube_srvc: minikube.service - - name: Start tps-tofino-ae.service + - name: Start TPS AE become: yes systemd: name: tps-tofino-ae.service @@ -43,28 +53,80 @@ debug: var: kube_pods_out.stdout_lines - - name: Install siddhi K8s operator - command: "kubectl apply -f {{ item }}" - loop: - - "{{ trans_sec_dir }}/playbooks/siddhi/kubernetes/kafka.yaml" - - "{{ trans_sec_dir }}/playbooks/siddhi/kubernetes/trpt-to-kafka.yaml" - - "{{ trans_sec_dir }}/playbooks/siddhi/kubernetes/kafka-trpt-ddos-detection.yaml" - - "{{ trans_sec_dir }}/playbooks/siddhi/kubernetes/kafka-trpt-drop-clear.yaml" + - name: Ensure {{ remote_scripts_dir }} scripts directory has been created + become: yes + file: + path: "{{ remote_scripts_dir }}" + state: directory + + - name: Query the Kafka Service clusterIP + command: kubectl get svc -n kafka my-cluster-kafka-bootstrap -o jsonpath="{.spec.clusterIPs[0]}" + register: cluster_ip_out - # TODO - Improve validation by for all containing "Running" - - name: Wait for K8s pods to come up... - command: kubectl get po - register: kube_pods_out + - name: Define the K8s CRDs + set_fact: + the_crds: + - "kafka-trpt-ddos-detection.yaml" + - "kafka-trpt-drop-clear.yaml" + - "trpt-to-kafka.yaml" + kafka_svc_ip: "{{ cluster_ip_out.stdout_lines[0] }}" + + - debug: + var: the_crds + + - name: Applying J2 templates of the siddhi K8s CRDs to {{ remote_scripts_dir }} with Kafka service IP - {{ kafka_svc_ip }} + become: yes + template: + src: "{{ trans_sec_dir }}/playbooks/siddhi/kubernetes/templates/{{ item }}.j2" + dest: "{{ remote_scripts_dir }}/{{ item }}" + loop: "{{ the_crds }}" + + - name: Install siddhi K8s SiddhiProcess + command: "kubectl apply -f {{ remote_scripts_dir }}/{{ item }}" + loop: "{{ the_crds }}" + + - name: Wait for K8s deployment/trpt-to-kafka-0 to become available + command: kubectl wait --for=condition=available --timeout=600s deployment/trpt-to-kafka-0 -n default + register: trpt_to_kafka_check retries: 10 + delay: 20 + until: trpt_to_kafka_check is not failed + + - name: Wait for K8s deployment/kafka-trpt-ddos-detect-0 to become available + command: kubectl wait --for=condition=available --timeout=600s deployment/kafka-trpt-ddos-detect-0 -n default + register: ddos_check + retries: 5 + delay: 10 + until: ddos_check is not failed + + - name: Wait for K8s deployment/kafka-trpt-drop-clear-0 to become available + command: kubectl wait --for=condition=available --timeout=600s deployment/kafka-trpt-drop-clear-0 -n default + register: drop_check + retries: 5 delay: 10 - until: kube_pods_out.stdout.find("Running") != -1 and - kube_pods_out.stdout.find("Creating") == -1 and - kube_pods_out.stdout_lines | length >= 9 + until: drop_check is not failed + + - name: Get pods in default namespace + command: kubectl get pods -n default + register: kube_pods_out - name: Show pods debug: var: kube_pods_out.stdout_lines + # TODO - Find a better means to obtain this bridge as things may break as soon as another interface prefaced with "br-" is created + - name: Query for the docker/K8s bridge + shell: "ip -o link show | awk -F': ' '{print $2}'| grep br-" + register: bridge_name_out + + - name: Store the CRDs + set_fact: + docker_bridge_name: "{{ bridge_name_out.stdout_lines[0] }}" + + # TODO - Reconsider this approach which was hacked in place as TRPTs aren't getting routed into the pod + - name: Forward all {{ clone_tun_name }} packets to the Docker/K8s service bridge {{ docker_bridge_name }} + command: "sudo iptables -A FORWARD -i {{ clone_tun_name }} -o {{ docker_bridge_name }} -j ACCEPT" + - name: Stop tps-tofino-ae.service become: yes systemd: diff --git a/playbooks/tofino/setup_nodes-lab_trial.yml b/playbooks/tofino/setup_nodes-lab_trial.yml index 7f343883..539cc9be 100644 --- a/playbooks/tofino/setup_nodes-lab_trial.yml +++ b/playbooks/tofino/setup_nodes-lab_trial.yml @@ -24,6 +24,7 @@ - import_playbook: setup_minikube_siddhi_ae.yml vars: host_val: ae + clone_tun_name: core-tun1 # Create virtual interfaces on switches - import_playbook: setup_virt_eth.yml @@ -51,3 +52,4 @@ port_to_wait: "{{ sdn_port }}" wait_timeout: 60 load_p4: False + diff --git a/playbooks/tofino/setup_tofino_switch.yml b/playbooks/tofino/setup_tofino_switch.yml index 669cf892..efd3dd5d 100644 --- a/playbooks/tofino/setup_tofino_switch.yml +++ b/playbooks/tofino/setup_tofino_switch.yml @@ -27,6 +27,7 @@ # Start tofino model chip emulator - import_playbook: ../general/start_service.yml vars: + setup_for_hw: "{{ from_hw | default(False) }}" host_val: "{{ switches }}" service_name: tps-tofino-model topo_dict: "{{ lookup('file','{{ topo_file_loc }}') | from_yaml }}" @@ -37,7 +38,7 @@ prog_name: "{{ p4_prog }}" additional_tmplt_file: "{{ trans_sec_dir }}/playbooks/tofino/templates/tofino-model-veth-port-mapping.json.j2" additional_tmplt_out_file: "{{ remote_scripts_dir }}/port-mapping.json" - when: not from_hw | bool + when: not setup_for_hw | bool # Start switchd - import_playbook: ../general/start_service.yml diff --git a/playbooks/tofino/setup_virt_eth.yml b/playbooks/tofino/setup_virt_eth.yml index a318d7ab..363428dc 100644 --- a/playbooks/tofino/setup_virt_eth.yml +++ b/playbooks/tofino/setup_virt_eth.yml @@ -20,10 +20,11 @@ SDE: "{{ remote_sde_dir }}" SDE_INSTALL: "{{ remote_sde_dir }}/install" vars: - setup_veth: "{{ not from_hw|bool | default(True) }}" + setup_for_hw: "{{ from_hw | default(False) }}" + setup_veth: "{{ not setup_for_hw | bool }}" tasks: - debug: - var: from_hw + var: setup_for_hw - debug: var: setup_veth - name: Setup DMA diff --git a/playbooks/tofino/templates/topology_template-lab_trial.yaml.j2 b/playbooks/tofino/templates/topology_template-lab_trial.yaml.j2 index a0c356a4..be25796e 100644 --- a/playbooks/tofino/templates/topology_template-lab_trial.yaml.j2 +++ b/playbooks/tofino/templates/topology_template-lab_trial.yaml.j2 @@ -73,7 +73,8 @@ hosts: {% if host_user is defined %} user: {{ host_user }} {% endif %} - public_ip: {{ ae_ip }} + public_ip: {{ ae_mgmt_ip }} + streaming_ip: {{ ae_ip }} tun1_ip: {{ ae_tun1_ip }} tun1_mac: {{ ae_tun1_mac }} tunnels: diff --git a/playbooks/tofino/tunnel/setup_gre_tunnels.yml b/playbooks/tofino/tunnel/setup_gre_tunnels.yml index b1190e6a..e326e644 100644 --- a/playbooks/tofino/tunnel/setup_gre_tunnels.yml +++ b/playbooks/tofino/tunnel/setup_gre_tunnels.yml @@ -19,10 +19,11 @@ gather_facts: yes become: yes vars: - setup_tunnels: "{{ not from_hw|bool | default(True) }}" + setup_for_hw: "{{ from_hw | default(False) }}" + setup_tunnels: "{{ not setup_for_hw | bool }}" tasks: - debug: - var: from_hw + var: setup_for_hw - debug: var: setup_tunnels - block: @@ -38,7 +39,7 @@ name: ip_gre state: present - - name: install ip_gre module + - name: Setup ipv4 forwarding lineinfile: path: /etc/sysctl.conf line: "net.ipv4.ip_forward=1" From d49bc9ec54bdaef4e114f3e30fef0826815f65fb Mon Sep 17 00:00:00 2001 From: spisarski Date: Fri, 23 Jul 2021 15:28:53 -0600 Subject: [PATCH 37/40] Initial changes for converting the TPS AE to run in K8s Created new AE image with minikube and dependencies installed Changed node setup for the lab_trial scenario to load the necessary deployments and siddhi operators for which there are still some issues --- .../kubernetes/kafka-trpt-ddos-detection.yaml | 68 +++++++++++ .../kubernetes/kafka-trpt-drop-clear.yaml | 65 +++++++++++ playbooks/siddhi/kubernetes/kafka.yaml | 110 ++++++++++++++++++ .../siddhi/kubernetes/trpt-to-kafka.yaml | 65 +++++++++++ playbooks/tofino/setup_nodes-lab_trial.yml | 1 - 5 files changed, 308 insertions(+), 1 deletion(-) create mode 100644 playbooks/siddhi/kubernetes/kafka-trpt-ddos-detection.yaml create mode 100644 playbooks/siddhi/kubernetes/kafka-trpt-drop-clear.yaml create mode 100644 playbooks/siddhi/kubernetes/kafka.yaml create mode 100644 playbooks/siddhi/kubernetes/trpt-to-kafka.yaml diff --git a/playbooks/siddhi/kubernetes/kafka-trpt-ddos-detection.yaml b/playbooks/siddhi/kubernetes/kafka-trpt-ddos-detection.yaml new file mode 100644 index 00000000..77f19f68 --- /dev/null +++ b/playbooks/siddhi/kubernetes/kafka-trpt-ddos-detection.yaml @@ -0,0 +1,68 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: siddhi.io/v1alpha2 +kind: SiddhiProcess +metadata: + name: kafka-trpt-ddos-detect +spec: + apps: + - script: | + @App:name("KafkaSourcePacketJSON") + @source( + type="kafka", + topic.list="trptPacket", + bootstrap.servers="kafka-service:9092", + group.id="test", + threading.option="single.thread", + @map( + type="p4-trpt", + @attributes( + src_mac="intHdr.mdStackHdr.origMac", + ip_ver="ipHdr.version", + dst_ip="ipHdr.dstAddr", + dst_port="protoHdr.dstPort" + ) + ) + ) + define stream trptPktStream ( + src_mac string, + ip_ver int, + dst_ip string, + dst_port long); + + @sink( + type="http", + publisher.url="http://localhost:9998/aggAttack", + method="POST", + headers="trp:headers", + @map(type="json") + ) + define stream attackStream ( + src_mac string, + ip_ver int, + dst_ip string, + dst_port long, + count long + ); + + @info(name = "trptJsonQuery") + from trptPktStream#window.time(1 sec) + select src_mac, ip_ver, dst_ip, dst_port, count(ip_ver) as count + group by src_mac, dst_ip, dst_port + having count == 10 + insert into attackStream; + + container: + image: "spisarski/siddhi" + imagePullPolicy: Always diff --git a/playbooks/siddhi/kubernetes/kafka-trpt-drop-clear.yaml b/playbooks/siddhi/kubernetes/kafka-trpt-drop-clear.yaml new file mode 100644 index 00000000..0be12801 --- /dev/null +++ b/playbooks/siddhi/kubernetes/kafka-trpt-drop-clear.yaml @@ -0,0 +1,65 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: siddhi.io/v1alpha2 +kind: SiddhiProcess +metadata: + name: kafka-trpt-drop-clear +spec: + apps: + - script: | + @App:name("KafkaSourceDropJSON") + @source( + type="kafka", + topic.list="trptDrop", + bootstrap.servers="kafka-service:9092", + group.id="test", + threading.option="single.thread", + @map( + type="p4-trpt", + @attributes( + timestamp="dropHdr.timestamp", + dropKey="dropHdr.dropKey", + dropCount="dropHdr.dropCount" + ) + ) + ) + define stream trptDropStream ( + timestamp long, + dropKey string, + dropCount long + ); + + @sink( + type="http", + publisher.url="http://localhost:9998/aggAttack", + method="DELETE", + headers="trp:headers", + @map(type="json") + ) + define stream dropAttackStream ( + dropKey string, + dropCount long, + count long + ); + + @info(name = "trptJsonQuery") + from trptDropStream#window.time(35 sec) + select dropKey, dropCount, count(dropCount) as count + group by dropKey, dropCount + having count >= 3 + insert into dropAttackStream; + + container: + image: "spisarski/siddhi" + imagePullPolicy: Always diff --git a/playbooks/siddhi/kubernetes/kafka.yaml b/playbooks/siddhi/kubernetes/kafka.yaml new file mode 100644 index 00000000..ad6b6629 --- /dev/null +++ b/playbooks/siddhi/kubernetes/kafka.yaml @@ -0,0 +1,110 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: zookeeper-deploy +spec: + replicas: 2 + selector: + matchLabels: + app: zookeeper-1 + template: + metadata: + labels: + app: zookeeper-1 + spec: + containers: + - name: zoo1 + image: digitalwonderland/zookeeper + ports: + - containerPort: 2181 + env: + - name: ZOOKEEPER_ID + value: "1" + - name: ZOOKEEPER_SERVER_1 + value: zoo1 + +--- +apiVersion: v1 +kind: Service +metadata: + name: zoo1 + labels: + app: zookeeper-1 +spec: + ports: + - name: client + port: 2181 + protocol: TCP + - name: follower + port: 2888 + protocol: TCP + - name: leader + port: 3888 + protocol: TCP + selector: + app: zookeeper-1 + +--- +apiVersion: v1 +kind: Service +metadata: + name: kafka-service + labels: + name: kafka +spec: + ports: + - port: 9092 + name: kafka-port + protocol: TCP + selector: + app: kafka + id: "0" + type: LoadBalancer + +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: kafka-broker0 +spec: + replicas: 2 + selector: + matchLabels: + app: kafka + id: "0" + template: + metadata: + labels: + app: kafka + id: "0" + spec: + containers: + - name: kafka + image: wurstmeister/kafka + ports: + - containerPort: 9092 + env: + - name: KAFKA_ADVERTISED_PORT + value: "9092" + - name: KAFKA_ADVERTISED_HOST_NAME + value: kafka + - name: KAFKA_ZOOKEEPER_CONNECT + value: zoo1:2181 + - name: KAFKA_BROKER_ID + value: "0" + - name: KAFKA_CREATE_TOPICS + value: admintome-test:1:1 diff --git a/playbooks/siddhi/kubernetes/trpt-to-kafka.yaml b/playbooks/siddhi/kubernetes/trpt-to-kafka.yaml new file mode 100644 index 00000000..1c240575 --- /dev/null +++ b/playbooks/siddhi/kubernetes/trpt-to-kafka.yaml @@ -0,0 +1,65 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: siddhi.io/v1alpha2 +kind: SiddhiProcess +metadata: + name: trpt-to-kafka +spec: + apps: + - script: | + @App:name("UDPSourceTRPT") + @source( + type="udp", + listen.port="556", + @map( + type="p4-trpt", + @attributes( + in_type="telemRptHdr.inType", + full_json="jsonString" + ) + ) + ) + define stream trptUdpStream (in_type int, full_json object); + + @sink( + type="kafka", + topic="trptPacket", + bootstrap.servers="kafka-service:9092", + is.binary.message = "false", + @map(type="text") + ) + define stream trptPacket (full_json object); + + @sink( + type="kafka", + topic="trptDrop", + bootstrap.servers="kafka-service:9092", + is.binary.message = "false", + @map(type="text") + ) + define stream trptDrop (full_json object); + + @info(name = "TrptPacket") + from trptUdpStream[in_type != 2] + select full_json + insert into trptPacket; + + @info(name = "TrptDrop") + from trptUdpStream[in_type == 2] + select full_json + insert into trptDrop; + + container: + image: "spisarski/siddhi" + imagePullPolicy: Always diff --git a/playbooks/tofino/setup_nodes-lab_trial.yml b/playbooks/tofino/setup_nodes-lab_trial.yml index 539cc9be..428deb18 100644 --- a/playbooks/tofino/setup_nodes-lab_trial.yml +++ b/playbooks/tofino/setup_nodes-lab_trial.yml @@ -52,4 +52,3 @@ port_to_wait: "{{ sdn_port }}" wait_timeout: 60 load_p4: False - From f80d54f654e11d8142942511019e6b3af0c2a112 Mon Sep 17 00:00:00 2001 From: spisarski Date: Fri, 30 Jul 2021 15:17:44 -0600 Subject: [PATCH 38/40] Example K8s CRDs --- .../kafka-trpt-ddos-detection.yaml.j2 | 0 .../templates/kafka-trpt-drop-clear.yaml.j2 | 0 .../templates/trpt-pv.yaml.j2 | 0 .../templates/trpt-to-kafka.yaml.j2 | 0 .../kubernetes/kafka-trpt-ddos-detection.yaml | 68 ----------- .../kubernetes/kafka-trpt-drop-clear.yaml | 65 ----------- playbooks/siddhi/kubernetes/kafka.yaml | 110 ------------------ .../siddhi/kubernetes/trpt-to-kafka.yaml | 65 ----------- 8 files changed, 308 deletions(-) rename {playbooks/siddhi/kubernetes => examples}/templates/kafka-trpt-ddos-detection.yaml.j2 (100%) rename {playbooks/siddhi/kubernetes => examples}/templates/kafka-trpt-drop-clear.yaml.j2 (100%) rename {playbooks/siddhi/kubernetes => examples}/templates/trpt-pv.yaml.j2 (100%) rename {playbooks/siddhi/kubernetes => examples}/templates/trpt-to-kafka.yaml.j2 (100%) delete mode 100644 playbooks/siddhi/kubernetes/kafka-trpt-ddos-detection.yaml delete mode 100644 playbooks/siddhi/kubernetes/kafka-trpt-drop-clear.yaml delete mode 100644 playbooks/siddhi/kubernetes/kafka.yaml delete mode 100644 playbooks/siddhi/kubernetes/trpt-to-kafka.yaml diff --git a/playbooks/siddhi/kubernetes/templates/kafka-trpt-ddos-detection.yaml.j2 b/examples/templates/kafka-trpt-ddos-detection.yaml.j2 similarity index 100% rename from playbooks/siddhi/kubernetes/templates/kafka-trpt-ddos-detection.yaml.j2 rename to examples/templates/kafka-trpt-ddos-detection.yaml.j2 diff --git a/playbooks/siddhi/kubernetes/templates/kafka-trpt-drop-clear.yaml.j2 b/examples/templates/kafka-trpt-drop-clear.yaml.j2 similarity index 100% rename from playbooks/siddhi/kubernetes/templates/kafka-trpt-drop-clear.yaml.j2 rename to examples/templates/kafka-trpt-drop-clear.yaml.j2 diff --git a/playbooks/siddhi/kubernetes/templates/trpt-pv.yaml.j2 b/examples/templates/trpt-pv.yaml.j2 similarity index 100% rename from playbooks/siddhi/kubernetes/templates/trpt-pv.yaml.j2 rename to examples/templates/trpt-pv.yaml.j2 diff --git a/playbooks/siddhi/kubernetes/templates/trpt-to-kafka.yaml.j2 b/examples/templates/trpt-to-kafka.yaml.j2 similarity index 100% rename from playbooks/siddhi/kubernetes/templates/trpt-to-kafka.yaml.j2 rename to examples/templates/trpt-to-kafka.yaml.j2 diff --git a/playbooks/siddhi/kubernetes/kafka-trpt-ddos-detection.yaml b/playbooks/siddhi/kubernetes/kafka-trpt-ddos-detection.yaml deleted file mode 100644 index 77f19f68..00000000 --- a/playbooks/siddhi/kubernetes/kafka-trpt-ddos-detection.yaml +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright (c) 2021 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: siddhi.io/v1alpha2 -kind: SiddhiProcess -metadata: - name: kafka-trpt-ddos-detect -spec: - apps: - - script: | - @App:name("KafkaSourcePacketJSON") - @source( - type="kafka", - topic.list="trptPacket", - bootstrap.servers="kafka-service:9092", - group.id="test", - threading.option="single.thread", - @map( - type="p4-trpt", - @attributes( - src_mac="intHdr.mdStackHdr.origMac", - ip_ver="ipHdr.version", - dst_ip="ipHdr.dstAddr", - dst_port="protoHdr.dstPort" - ) - ) - ) - define stream trptPktStream ( - src_mac string, - ip_ver int, - dst_ip string, - dst_port long); - - @sink( - type="http", - publisher.url="http://localhost:9998/aggAttack", - method="POST", - headers="trp:headers", - @map(type="json") - ) - define stream attackStream ( - src_mac string, - ip_ver int, - dst_ip string, - dst_port long, - count long - ); - - @info(name = "trptJsonQuery") - from trptPktStream#window.time(1 sec) - select src_mac, ip_ver, dst_ip, dst_port, count(ip_ver) as count - group by src_mac, dst_ip, dst_port - having count == 10 - insert into attackStream; - - container: - image: "spisarski/siddhi" - imagePullPolicy: Always diff --git a/playbooks/siddhi/kubernetes/kafka-trpt-drop-clear.yaml b/playbooks/siddhi/kubernetes/kafka-trpt-drop-clear.yaml deleted file mode 100644 index 0be12801..00000000 --- a/playbooks/siddhi/kubernetes/kafka-trpt-drop-clear.yaml +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright (c) 2021 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: siddhi.io/v1alpha2 -kind: SiddhiProcess -metadata: - name: kafka-trpt-drop-clear -spec: - apps: - - script: | - @App:name("KafkaSourceDropJSON") - @source( - type="kafka", - topic.list="trptDrop", - bootstrap.servers="kafka-service:9092", - group.id="test", - threading.option="single.thread", - @map( - type="p4-trpt", - @attributes( - timestamp="dropHdr.timestamp", - dropKey="dropHdr.dropKey", - dropCount="dropHdr.dropCount" - ) - ) - ) - define stream trptDropStream ( - timestamp long, - dropKey string, - dropCount long - ); - - @sink( - type="http", - publisher.url="http://localhost:9998/aggAttack", - method="DELETE", - headers="trp:headers", - @map(type="json") - ) - define stream dropAttackStream ( - dropKey string, - dropCount long, - count long - ); - - @info(name = "trptJsonQuery") - from trptDropStream#window.time(35 sec) - select dropKey, dropCount, count(dropCount) as count - group by dropKey, dropCount - having count >= 3 - insert into dropAttackStream; - - container: - image: "spisarski/siddhi" - imagePullPolicy: Always diff --git a/playbooks/siddhi/kubernetes/kafka.yaml b/playbooks/siddhi/kubernetes/kafka.yaml deleted file mode 100644 index ad6b6629..00000000 --- a/playbooks/siddhi/kubernetes/kafka.yaml +++ /dev/null @@ -1,110 +0,0 @@ -# Copyright (c) 2021 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -kind: Deployment -apiVersion: apps/v1 -metadata: - name: zookeeper-deploy -spec: - replicas: 2 - selector: - matchLabels: - app: zookeeper-1 - template: - metadata: - labels: - app: zookeeper-1 - spec: - containers: - - name: zoo1 - image: digitalwonderland/zookeeper - ports: - - containerPort: 2181 - env: - - name: ZOOKEEPER_ID - value: "1" - - name: ZOOKEEPER_SERVER_1 - value: zoo1 - ---- -apiVersion: v1 -kind: Service -metadata: - name: zoo1 - labels: - app: zookeeper-1 -spec: - ports: - - name: client - port: 2181 - protocol: TCP - - name: follower - port: 2888 - protocol: TCP - - name: leader - port: 3888 - protocol: TCP - selector: - app: zookeeper-1 - ---- -apiVersion: v1 -kind: Service -metadata: - name: kafka-service - labels: - name: kafka -spec: - ports: - - port: 9092 - name: kafka-port - protocol: TCP - selector: - app: kafka - id: "0" - type: LoadBalancer - ---- -kind: Deployment -apiVersion: apps/v1 -metadata: - name: kafka-broker0 -spec: - replicas: 2 - selector: - matchLabels: - app: kafka - id: "0" - template: - metadata: - labels: - app: kafka - id: "0" - spec: - containers: - - name: kafka - image: wurstmeister/kafka - ports: - - containerPort: 9092 - env: - - name: KAFKA_ADVERTISED_PORT - value: "9092" - - name: KAFKA_ADVERTISED_HOST_NAME - value: kafka - - name: KAFKA_ZOOKEEPER_CONNECT - value: zoo1:2181 - - name: KAFKA_BROKER_ID - value: "0" - - name: KAFKA_CREATE_TOPICS - value: admintome-test:1:1 diff --git a/playbooks/siddhi/kubernetes/trpt-to-kafka.yaml b/playbooks/siddhi/kubernetes/trpt-to-kafka.yaml deleted file mode 100644 index 1c240575..00000000 --- a/playbooks/siddhi/kubernetes/trpt-to-kafka.yaml +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright (c) 2021 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: siddhi.io/v1alpha2 -kind: SiddhiProcess -metadata: - name: trpt-to-kafka -spec: - apps: - - script: | - @App:name("UDPSourceTRPT") - @source( - type="udp", - listen.port="556", - @map( - type="p4-trpt", - @attributes( - in_type="telemRptHdr.inType", - full_json="jsonString" - ) - ) - ) - define stream trptUdpStream (in_type int, full_json object); - - @sink( - type="kafka", - topic="trptPacket", - bootstrap.servers="kafka-service:9092", - is.binary.message = "false", - @map(type="text") - ) - define stream trptPacket (full_json object); - - @sink( - type="kafka", - topic="trptDrop", - bootstrap.servers="kafka-service:9092", - is.binary.message = "false", - @map(type="text") - ) - define stream trptDrop (full_json object); - - @info(name = "TrptPacket") - from trptUdpStream[in_type != 2] - select full_json - insert into trptPacket; - - @info(name = "TrptDrop") - from trptUdpStream[in_type == 2] - select full_json - insert into trptDrop; - - container: - image: "spisarski/siddhi" - imagePullPolicy: Always From a16f7f76ac65d2085719734a0a38500bd279982e Mon Sep 17 00:00:00 2001 From: spisarski Date: Mon, 2 Aug 2021 13:10:03 -0600 Subject: [PATCH 39/40] Able to detect attacks but drop packets are not getting to the AE --- .../kafka-trpt-ddos-detection.yaml.j2 | 87 +++++++++++++++++++ .../templates/kafka-trpt-drop-clear.yaml.j2 | 84 ++++++++++++++++++ .../kubernetes/templates/trpt-pv.yaml.j2 | 15 ++++ .../templates/trpt-to-kafka.yaml.j2 | 84 ++++++++++++++++++ playbooks/tofino/setup_nodes-lab_trial.yml | 1 + 5 files changed, 271 insertions(+) create mode 100644 playbooks/siddhi/kubernetes/templates/kafka-trpt-ddos-detection.yaml.j2 create mode 100644 playbooks/siddhi/kubernetes/templates/kafka-trpt-drop-clear.yaml.j2 create mode 100644 playbooks/siddhi/kubernetes/templates/trpt-pv.yaml.j2 create mode 100644 playbooks/siddhi/kubernetes/templates/trpt-to-kafka.yaml.j2 diff --git a/playbooks/siddhi/kubernetes/templates/kafka-trpt-ddos-detection.yaml.j2 b/playbooks/siddhi/kubernetes/templates/kafka-trpt-ddos-detection.yaml.j2 new file mode 100644 index 00000000..a8dcb0b1 --- /dev/null +++ b/playbooks/siddhi/kubernetes/templates/kafka-trpt-ddos-detection.yaml.j2 @@ -0,0 +1,87 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: siddhi.io/v1alpha2 +kind: SiddhiProcess +metadata: + name: kafka-trpt-ddos-detect +spec: + apps: + - script: | + @App:name("KafkaSourcePacketJSON") + @source( + type="kafka", + topic.list="trptPacket", + bootstrap.servers="{{ kafka_svc_ip }}:9092", + group.id="test", + threading.option="single.thread", + @map( + type="p4-trpt", + @attributes( + src_mac="intHdr.mdStackHdr.origMac", + ip_ver="ipHdr.version", + dst_ip="ipHdr.dstAddr", + dst_port="protoHdr.dstPort" + ) + ) + ) + define stream trptPktStream ( + src_mac string, + ip_ver int, + dst_ip string, + dst_port long); + + @sink( + type="http", + publisher.url="http://{{ sdn_ip }}:9998/aggAttack", + method="POST", + headers="trp:headers", + @map(type="json") + ) + define stream attackStream ( + src_mac string, + ip_ver int, + dst_ip string, + dst_port long, + count long + ); + + @info(name = "trptJsonQuery") + from trptPktStream#window.time(1 sec) + select src_mac, ip_ver, dst_ip, dst_port, count(ip_ver) as count + group by src_mac, dst_ip, dst_port + having count == 10 + insert into attackStream; + + container: + image: "spisarski/siddhi" + imagePullPolicy: Always + + persistentVolumeClaim: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: standard + volumeMode: Filesystem + + runner: | + state.persistence: + enabled: true + intervalInMin: 5 + revisionsToKeep: 2 + persistenceStore: io.siddhi.distribution.core.persistence.FileSystemPersistenceStore + config: + location: siddhi-app-persistence diff --git a/playbooks/siddhi/kubernetes/templates/kafka-trpt-drop-clear.yaml.j2 b/playbooks/siddhi/kubernetes/templates/kafka-trpt-drop-clear.yaml.j2 new file mode 100644 index 00000000..3f80d227 --- /dev/null +++ b/playbooks/siddhi/kubernetes/templates/kafka-trpt-drop-clear.yaml.j2 @@ -0,0 +1,84 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: siddhi.io/v1alpha2 +kind: SiddhiProcess +metadata: + name: kafka-trpt-drop-clear +spec: + apps: + - script: | + @App:name("KafkaSourceDropJSON") + @source( + type="kafka", + topic.list="trptDrop", + bootstrap.servers="{{ kafka_svc_ip }}:9092", + group.id="test", + threading.option="single.thread", + @map( + type="p4-trpt", + @attributes( + timestamp="dropHdr.timestamp", + dropKey="dropHdr.dropKey", + dropCount="dropHdr.dropCount" + ) + ) + ) + define stream trptDropStream ( + timestamp long, + dropKey string, + dropCount long + ); + + @sink( + type="http", + publisher.url="http://{{ sdn_ip }}:9998/aggAttack", + method="DELETE", + headers="trp:headers", + @map(type="json") + ) + define stream dropAttackStream ( + dropKey string, + dropCount long, + count long + ); + + @info(name = "trptJsonQuery") + from trptDropStream#window.time(35 sec) + select dropKey, dropCount, count(dropCount) as count + group by dropKey, dropCount + having count >= 3 + insert into dropAttackStream; + + container: + image: "spisarski/siddhi" + imagePullPolicy: Always + + persistentVolumeClaim: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: standard + volumeMode: Filesystem + + runner: | + state.persistence: + enabled: true + intervalInMin: 5 + revisionsToKeep: 2 + persistenceStore: io.siddhi.distribution.core.persistence.FileSystemPersistenceStore + config: + location: siddhi-app-persistence diff --git a/playbooks/siddhi/kubernetes/templates/trpt-pv.yaml.j2 b/playbooks/siddhi/kubernetes/templates/trpt-pv.yaml.j2 new file mode 100644 index 00000000..f21afb6f --- /dev/null +++ b/playbooks/siddhi/kubernetes/templates/trpt-pv.yaml.j2 @@ -0,0 +1,15 @@ +kind: PersistentVolume +apiVersion: v1 +metadata: + name: siddhi-pv + labels: + type: local +spec: + storageClassName: standard + persistentVolumeReclaimPolicy: Delete + capacity: + storage: 1Gi + accessModes: + - ReadWriteOnce + hostPath: + path: "/home/siddhi_user/" diff --git a/playbooks/siddhi/kubernetes/templates/trpt-to-kafka.yaml.j2 b/playbooks/siddhi/kubernetes/templates/trpt-to-kafka.yaml.j2 new file mode 100644 index 00000000..8bb5a190 --- /dev/null +++ b/playbooks/siddhi/kubernetes/templates/trpt-to-kafka.yaml.j2 @@ -0,0 +1,84 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: siddhi.io/v1alpha2 +kind: SiddhiProcess +metadata: + name: trpt-to-kafka +spec: + apps: + - script: | + @App:name("UDPSourceTRPT") + @source( + type="udp", + listen.port="556", + @map( + type="p4-trpt", + @attributes( + in_type="telemRptHdr.inType", + full_json="jsonString" + ) + ) + ) + define stream trptUdpStream (in_type int, full_json object); + + @sink( + type="kafka", + topic="trptPacket", + bootstrap.servers="{{ kafka_svc_ip }}:9092", + is.binary.message = "false", + @map(type="text") + ) + define stream trptPacket (full_json object); + + @sink( + type="kafka", + topic="trptDrop", + bootstrap.servers="{{ kafka_svc_ip }}:9092", + is.binary.message = "false", + @map(type="text") + ) + define stream trptDrop (full_json object); + + @info(name = "TrptPacket") + from trptUdpStream[in_type != 2] + select full_json + insert into trptPacket; + + @info(name = "TrptDrop") + from trptUdpStream[in_type == 2] + select full_json + insert into trptDrop; + + container: + image: "spisarski/siddhi" + imagePullPolicy: Always + +--- +apiVersion: v1 +kind: Service +metadata: + name: trpt-to-kafka-0 +spec: + type: LoadBalancer + clusterIP: {{ ae_ip }} + selector: + siddhi.io/instance: trpt-to-kafka-0 + siddhi.io/name: SiddhiProcess + siddhi.io/part-of: siddhi-operator + siddhi.io/version: 0.2.2 + ports: + - port: 556 + targetPort: 556 + protocol: UDP diff --git a/playbooks/tofino/setup_nodes-lab_trial.yml b/playbooks/tofino/setup_nodes-lab_trial.yml index 428deb18..539cc9be 100644 --- a/playbooks/tofino/setup_nodes-lab_trial.yml +++ b/playbooks/tofino/setup_nodes-lab_trial.yml @@ -52,3 +52,4 @@ port_to_wait: "{{ sdn_port }}" wait_timeout: 60 load_p4: False + From f4bc555276a31d55611aa03ade5576e465a95b4e Mon Sep 17 00:00:00 2001 From: spisarski Date: Wed, 18 Aug 2021 12:56:23 -0600 Subject: [PATCH 40/40] Fix merge issue & create new Siddhi image as upstream custom extensions had minor changes. --- automation/p4/tofino/setup.tf | 1 - automation/p4/tofino/variables.tf | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/automation/p4/tofino/setup.tf b/automation/p4/tofino/setup.tf index 588a7762..e7663cb9 100644 --- a/automation/p4/tofino/setup.tf +++ b/automation/p4/tofino/setup.tf @@ -21,7 +21,6 @@ locals { agg_switch_ip = var.scenario_name == "lab_trial" ? aws_instance.tps-switch.1.private_ip: "n/a" agg_tun1_ip = var.scenario_name == "lab_trial" ? aws_network_interface.switch_tun_1.1.private_ip: "n/a" agg_tun1_mac = var.scenario_name == "lab_trial" ? aws_network_interface.switch_tun_1.1.mac_address: "n/a" - ae_ip = var.scenario_name == "lab_trial" ? aws_instance.ae.private_ip: "n/a" ae_ip = var.scenario_name == "lab_trial" ? var.ae_k8s_svc_ip: "n/a" ae_mgmt_ip = var.scenario_name == "lab_trial" ? aws_instance.ae.private_ip: "n/a" ae_tun1_ip = var.scenario_name == "lab_trial" ? aws_network_interface.ae_tun_1.private_ip: "n/a" diff --git a/automation/p4/tofino/variables.tf b/automation/p4/tofino/variables.tf index cb0daffc..478ad348 100644 --- a/automation/p4/tofino/variables.tf +++ b/automation/p4/tofino/variables.tf @@ -35,7 +35,7 @@ variable "tofino" { } } -variable "siddhi_ae_ami" {default = "ami-07e5e73831d02dbe9"} +variable "siddhi_ae_ami" {default = "ami-010458ceb325ec348"} variable "switch_instance_type" {default = "t2.2xlarge"} variable "ae_instance_type" {default = "t2.2xlarge"}