diff --git a/README.md b/README.md index 1167a20b..062afa89 100644 --- a/README.md +++ b/README.md @@ -31,3 +31,23 @@ We use an [Apache 2.0 License](LICENSE) for Transparent Security. Questions? Just send us an email at [transparent-security@cablelabs.com](mailto:transparent-security@cablelabs.com) or [open an issue](https://github.com/cablelabs/transparent-security/issues). + +## The docs +These directories contain other documentation +- [Analytic Engine](./docs/ae) - [How to setup the AE](./docs/ae/SIDDHI_AE_SETUP.md) + - [kubernetes](./docs/ae/kubernetes) - directory containing sample CRDs to deploy the TPS AE on Kubernetes +- [Build automation AMIs](./docs/env_build) - [Creating the required EC2 Images](./docs/env_build/CREATE_AUTOMATION_IMAGES.md) +- [P4 INT](./docs/int_header) - [Description of the P4 INT header added to packets and associated Wireshark plugin](./docs/int_header/README.md) +- [Telemetry Report](./docs/telemetry_report) - [Description of the Telemetry Report UDP Packet](./docs/telemetry_report/telemetry_report.md) +- [Terraform Example Variable File](./docs/terraform) - Example "tfvars" files for configuring a Terraform run +- [P4 Automation](./docs/tofino) - [Instructions on how to execute the P4 automation scripts](./docs/tofino/RUN_CI_AUTOMATION.md) + +## The directories +- [automation](automation) - contains Terraform scripts for CI and testing on AWS +- [bin](bin) - miscellaneous scripts mostly used by scripts in automation +- [conf](conf) - miscellaneous environment configurations +- [docs](docs) - miscellaneous MD files +- [p4](p4) - The P4 source code +- [playbooks](playbooks) - The Ansible Playbooks used by automation +- [tests](tests) - the Python unit test directory +- [trans_sec](trans_sec) - the project's top-level Python package diff --git a/analytics/OpenDistroElasticSearch/Readme.md b/analytics/OpenDistroElasticSearch/Readme.md deleted file mode 100644 index b71cc0d7..00000000 --- a/analytics/OpenDistroElasticSearch/Readme.md +++ /dev/null @@ -1,4 +0,0 @@ -# Open Distro Elasticsearch Guides - -- To visualize and injest data use the [Data Mapping and Parsing Guide](/udp_data_mapping_and_parsing_pipeline.md) -- To detect DDOS attack and mitigate it use the [DDOS Detection and Mitigation Guide](ddos_detection_and_mitigation_pipeline.md) diff --git a/analytics/OpenDistroElasticSearch/anomaly_detector_config.json b/analytics/OpenDistroElasticSearch/anomaly_detector_config.json deleted file mode 100644 index 7312cc76..00000000 --- a/analytics/OpenDistroElasticSearch/anomaly_detector_config.json +++ /dev/null @@ -1,80 +0,0 @@ -{ - "name": "ddos-api-anomaly-detector", - "description": "DDOS API Anomaly Detector", - "time_field": "timestamp", - "indices": [ - "packets-*" - ], - "feature_attributes": [ - { - "feature_name": "UDP2DstPortValueCount", - "feature_enabled": true, - "aggregation_query": { - "UDP2DstPortValueCount": { - "value_count": { - "field": "ts.UDP2DstPort" - } - } - } - }, - { - "feature_name": "IPVDstAddrValueCount", - "feature_enabled": true, - "aggregation_query": { - "IPVDstAddrValueCount": { - "value_count": { - "field": "ts.IPvDestAddr" - } - } - } - }, - { - "feature_name": "OriginatingMacAddressValueCount", - "feature_enabled": true, - "aggregation_query": { - "OriginatingMacAddressValueCount": { - "value_count": { - "field": "ts.INTMetadataSourceMetadataOriginatingMac" - } - } - } - } - ], - "filter_query": { - "bool": { - "filter": [ - { - "term": { - "ts.IPVersion": { - "value": "4", - "boost": 1 - } - } - }, - { - "wildcard": { - "ts.INTMetadataSourceMetadataOriginatingMac": { - "wildcard": "*000000000101*", - "boost": 1 - } - } - } - ], - "adjust_pure_negative": true, - "boost": 1 - } - }, - "detection_interval": { - "period": { - "interval": 1, - "unit": "Minutes" - } - }, - "window_delay": { - "period": { - "interval": 1, - "unit": "Minutes" - } - }, - "shingle_size" : 1 -} diff --git a/analytics/OpenDistroElasticSearch/data_mapping.json b/analytics/OpenDistroElasticSearch/data_mapping.json deleted file mode 100644 index c919b6c7..00000000 --- a/analytics/OpenDistroElasticSearch/data_mapping.json +++ /dev/null @@ -1,264 +0,0 @@ -{ - "index_patterns": [ - "packets-*" - ], - "settings": { - "index.number_of_replicas": "0", - "index.max_script_fields": "200", - "index.default_pipeline": "ts_parsing" - }, - "mappings": { - "dynamic": "false", - "properties": { - "timestamp": { - "type": "date" - }, - "ts": { - "properties": { - "rawData": { - "type": "keyword" - }, - "tsRawDataSize": { - "type": "keyword" - }, - "udpLayersSize": { - "type": "keyword" - }, - "telemetryReport": { - "type": "keyword" - }, - "telemetryReportVersion": { - "type": "keyword" - }, - "telemetryHwdId": { - "type": "keyword" - }, - "telemetrySequenceNumber": { - "type": "keyword" - }, - "telemetryNodeId": { - "type": "keyword" - }, - "telemetryReportType": { - "type": "keyword" - }, - "telemetryInType": { - "type": "keyword" - }, - "telemetryReportLength": { - "type": "keyword" - }, - "telemetryMDLength": { - "type": "keyword" - }, - "telemetryD": { - "type": "keyword" - }, - "telemetryQ": { - "type": "keyword" - }, - "telemetryF": { - "type": "keyword" - }, - "telemetryI": { - "type": "keyword" - }, - "telemetryRsvd": { - "type": "keyword" - }, - "telemetryRepMdBits": { - "type": "keyword" - }, - "telemetryDomainSpecificID": { - "type": "keyword" - }, - "telemetryDsMdBits": { - "type": "keyword" - }, - "telemetryDsMdStatus": { - "type": "keyword" - }, - "telemetryVarOpt": { - "type": "keyword" - }, - "data": { - "type": "keyword" - }, - "ethernetHeaderDestMac": { - "type": "keyword" - }, - "ethernetHeaderSrcMac": { - "type": "keyword" - }, - "ethernetHeaderEtherType": { - "type": "keyword" - }, - "IPvData": { - "type": "keyword" - }, - "isIpv6": { - "type": "keyword" - }, - "isIpv4": { - "type": "keyword" - }, - "IPVersion": { - "type": "keyword" - }, - "IPvClass": { - "type": "keyword" - }, - "IPvFlow": { - "type": "keyword" - }, - "IPvPayloadLength": { - "type": "keyword" - }, - "IPvNextHdrProto": { - "type": "keyword" - }, - "IPvNextHopLimit": { - "type": "keyword" - }, - "IPvSrcAddr": { - "type": "keyword" - }, - "IPvDestAddr": { - "type": "keyword" - }, - "UDGPData": { - "type": "keyword" - }, - "UDGSrcPort": { - "type": "keyword" - }, - "UDGDstPort": { - "type": "keyword" - }, - "UDGLength": { - "type": "keyword" - }, - "UDGChecksum": { - "type": "keyword" - }, - "InBandNetworkTelemetryData": { - "type": "keyword" - }, - "UDPIntShimHeaderData": { - "type": "keyword" - }, - "UDPIntShimHeaderType": { - "type": "keyword" - }, - "UDPIntShimHeaderNPT": { - "type": "keyword" - }, - "UDPIntShimHeaderRes1": { - "type": "keyword" - }, - "UDPIntShimHeaderLength": { - "type": "keyword" - }, - "UDPIntShimHeaderRes2": { - "type": "keyword" - }, - "UDPIntShimHeaderNextProto": { - "type": "keyword" - }, - "INTMetadataHeaderData": { - "type": "keyword" - }, - "UDPIntMetaDataHeaderVersion": { - "type": "keyword" - }, - "UDPIntMetaDataHeaderRes": { - "type": "keyword" - }, - "UDPIntMetaDataHeaderD": { - "type": "keyword" - }, - "UDPIntMetaDataHeaderE": { - "type": "keyword" - }, - "UDPIntMetaDataHeaderM": { - "type": "keyword" - }, - "UDPIntMetaDataHeaderPerhopMetadataLength": { - "type": "keyword" - }, - "UDPIntMetaDataHeaderRemainingHopCnt": { - "type": "keyword" - }, - "UDPIntMetaDataHeaderInstructionBitmap": { - "type": "keyword" - }, - "UDPIntMetaDataHeaderDomainSpecificID": { - "type": "keyword" - }, - "UDPIntMetaDataHeaderDSInstruction": { - "type": "keyword" - }, - "UDPIntMetaDataHeaderDSFlags": { - "type": "keyword" - }, - "INTMetadataStackData": { - "type": "keyword" - }, - "INTMetadataStackSwitchID": { - "type": "keyword" - }, - "INTMetadataSourceMetadata": { - "type": "keyword" - }, - "INTMetadataSourceMetadataSwitchId": { - "type": "keyword" - }, - "INTMetadataSourceMetadataOriginatingMac": { - "type": "keyword" - }, - "INTMetadataSourceMetadataReserved": { - "type": "keyword" - }, - "UDP2Data": { - "type": "keyword" - }, - "UDP2SrcPort": { - "type": "keyword" - }, - "UDP2DstPort": { - "type": "keyword" - }, - "UDP2Length": { - "type": "keyword" - }, - "UDP2Checksum": { - "type": "keyword" - }, - "dataII": { - "type": "keyword" - } - } - }, - "layers": { - "properties": { - "data_raw": { - "type": "keyword" - }, - "data": { - "properties": { - "data_data_data_raw": { - "type": "keyword" - }, - "data_data_data": { - "type": "keyword" - }, - "data_data_len": { - "type": "keyword" - } - } - } - } - } - } - } -} diff --git a/analytics/OpenDistroElasticSearch/ddos_detection_and_mitigation_pipeline.md b/analytics/OpenDistroElasticSearch/ddos_detection_and_mitigation_pipeline.md deleted file mode 100644 index ebf8341f..00000000 --- a/analytics/OpenDistroElasticSearch/ddos_detection_and_mitigation_pipeline.md +++ /dev/null @@ -1,360 +0,0 @@ -# DDOS Detection and Mitigation Pipeline ( UDPv4 ) using Open Distro Elasticsearch - -- To add a DDOS Detection pipeline you need to add a monitor and an associated trigger to detect the DDOS attack. -- To add a DDOS Mitigation pipeline you need add an associated action (custom webhook) with associated trigger. - -The same can be done using the Kibana UI or programmatically using the [Anomaloy Detection API]( https://opendistro.github.io/for-elasticsearch-docs/docs/ad/api/ ) and [Alerting API]( https://opendistro.github.io/for-elasticsearch-docs/docs/alerting/api/ ) - -The below guides talks about using the Anomaloy Detection API and Alerting API. - -> **_NOTE:_** -The guide is specific to DDOS use case. Please change the Anomaloy Detectors, Monitors, Triggers and Actions according to your use case. -[Official Guide](https://opendistro.github.io/for-elasticsearch-docs/) - -## Add Anomaloy Detector -To create an Anomaloy Detector using the Kibana UI please look at [Official Guide](https://opendistro.github.io/for-elasticsearch-docs/docs/ad/) for more details. - -Use below anomaly detection operations to programmatically create and manage detectors. - -### Step 1: Create Anomaloy Detector - -- Push it -``` -curl -H 'Content-Type: application/json' -XPOST 'http://localhost:9200/_opendistro/_anomaly_detection/detectors' -d @anomaly_detector_config.json -``` -- Expected Response -``` -{ - "_id" : "UDkwkXUBenaD-0rLhn9b", - "_version" : 1, - "_seq_no" : 5, - "_primary_term" : 1, - "anomaly_detector" : { - "name" : "ddos-anomaly-detector", - "description" : "DDOS Anomaly Detector", - "time_field" : "timestamp", - "indices" : [ - "packets-*" - ], - "filter_query" : { - "bool" : { - "filter" : [ - { - "term" : { - "ts.IPVersion" : { - "value" : "4", - "boost" : 1.0 - } - } - }, - { - "wildcard" : { - "ts.INTMetadataSourceMetadataOriginatingMac" : { - "wildcard" : "*000000000101*", - "boost" : 1.0 - } - } - } - ], - "adjust_pure_negative" : true, - "boost" : 1.0 - } - }, - "detection_interval" : { - "period" : { - "interval" : 1, - "unit" : "Minutes" - } - }, - "window_delay" : { - "period" : { - "interval" : 1, - "unit" : "Minutes" - } - }, - "shingle_size" : 1, - "schema_version" : 0, - "feature_attributes" : [ - { - "feature_id" : "STkwkXUBenaD-0rLhn9Z", - "feature_name" : "UDP2DstPortValueCount", - "feature_enabled" : true, - "aggregation_query" : { - "UDP2DstPortValueCount" : { - "value_count" : { - "field" : "ts.UDP2DstPort" - } - } - } - }, - { - "feature_id" : "SjkwkXUBenaD-0rLhn9Z", - "feature_name" : "IPVDstAddrValueCount", - "feature_enabled" : true, - "aggregation_query" : { - "IPVDstAddrValueCount" : { - "value_count" : { - "field" : "ts.IPvDestAddr" - } - } - } - }, - { - "feature_id" : "SzkwkXUBenaD-0rLhn9Z", - "feature_name" : "OriginatingMacAddressValueCount", - "feature_enabled" : true, - "aggregation_query" : { - "OriginatingMacAddressValueCount" : { - "value_count" : { - "field" : "ts.INTMetadataSourceMetadataOriginatingMac" - } - } - } - } - ] - } -} -``` -- Verify it worked - -The following GET call with appropriate _id can be made to check the newly created anomaly detector. The _id can be found in reponse of Step 1 - -``` -curl -H 'Content-Type: application/json' -XPOST 'http://localhost:9200/_opendistro/_anomaly_detection/detectors/{_id} -``` - -- Start Anomaloy Detector - -``` -curl -H 'Content-Type: application/json' -XPOST 'http://localhost:9200/_opendistro/_anomaly_detection/detectors/{_id}/_start -``` - -### Step 2: Create Monitor - -A monitor is a job that runs on a defined schedule and queries Elasticsearch. The results of these queries are then used as input for one or more triggers. - -- Create custom SDN Webhook Alert Action to be associated with the Monitor - -``` -curl -H 'Content-Type: application/json' -XPOST 'http://localhost:9200/_opendistro/_alerting/destinations -d -'{ - "type": "custom_webhook", - "name": "SDN_Webhook", - "custom_webhook": { - "path": "/aggAttack", - "header_params": { - "Content-Type": "application/json" - }, - "scheme": "HTTP", - "port": 9998, - "host": "tps.sdn.org" - } -}' -``` -- Expected Response -``` -{ - "_id" : "jjlCkXUBenaD-0rLO4Cg", - "_version" : 1, - "_seq_no" : 16, - "_primary_term" : 1, - "destination" : { - "type" : "custom_webhook", - "name" : "SDN_Webhook", - "schema_version" : 2, - "last_update_time" : 1604459838368, - "custom_webhook" : { - "path" : "/aggAttack", - "header_params" : { - "Content-Type" : "application/json" - }, - "password" : null, - "scheme" : "HTTP", - "port" : 9998, - "query_params" : { }, - "host" : "tps.sdn.org", - "url" : null, - "username" : null - } - } -} -``` - -> **_NOTE:_** -> Edit the /etc/hosts file on analytics instance to add the ubuntu private IP for dns name resolution to tps.sdn.org or any other preferred dns name. The DNS name added in hosts file should correspond to host properties in above payload. -> The _id from response will be utilized when creating the associated Monitor. - -- Create Monitor - -``` -curl -H 'Content-Type: application/json' -XPOST 'http://localhost:9200/ _opendistro/_alerting/monitors -d -'{ - "type": "monitor", - "name": "DDOS-Monitor", - "enabled": true, - "schedule": { - "period": { - "interval": 1, - "unit": "MINUTES" - } - }, - "inputs": [{ - "search": { - "indices": ["packets-*"], - "query": { - "size": 0, - "query": { - "bool": { - "filter": [ - { - "range": { - "timestamp": { - "from": "{{period_end}}||-1m", - "to": "{{period_end}}", - "include_lower": true, - "include_upper": true, - "format": "epoch_millis", - "boost": 1 - } - } - }, - { - "term": { - "ts.IPVersion": { - "value": "4", - "boost": 1 - } - } - }, - { - "wildcard": { - "ts.IPvDestAddr": { - "wildcard": "*c0a8010a*", - "boost": 1 - } - } - }, - { - "wildcard": { - "ts.UDP2DstPort": { - "wildcard": "*5792*", - "boost": 1 - } - } - } - ], - "adjust_pure_negative": true, - "boost": 1 - } - }, - "_source": { - "includes": [ - "ts.UDP2DstPort", - "ts.IPvDestAddr", - "ts.INTMetadataSourceMetadataOriginatingMac", - "ts.IPv4SrcIP", - "ts.IPv4DestIP", - "ts.IPv6SrcIP", - "ts.IPv6DestIP" - ], - "excludes": [] - }, - "aggregations": { - "UDP2DstPortValueCount": { - "value_count": { - "field": "ts.UDP2DstPort" - } - }, - "IPvDestAddrValueCount": { - "value_count": { - "field": "ts.IPvDestAddr" - } - }, - "INTMetadataSourceMetadataOriginatingMacValueCount": { - "value_count": { - "field": "ts.INTMetadataSourceMetadataOriginatingMac" - } - }, - "OriginatingMacAddrUDPv4": { - "terms": { - "field": "ts.INTMetadataSourceMetadataOriginatingMac", - "size": 10, - "min_doc_count": 1, - "shard_min_doc_count": 0, - "show_term_doc_count_error": false, - "order": [ - { - "_count": "desc" - }, - { - "_key": "asc" - } - ] - } - }, - "IPv4SrcIP": { - "terms": { - "field": "ts.IPv4SrcIP", - "size": 10, - "min_doc_count": 1, - "shard_min_doc_count": 0, - "show_term_doc_count_error": false, - "order": [ - { - "_count": "desc" - }, - { - "_key": "asc" - } - ] - } - }, - "IPv4DestIP": { - "terms": { - "field": "ts.IPv4DestIP", - "size": 10, - "min_doc_count": 1, - "shard_min_doc_count": 0, - "show_term_doc_count_error": false, - "order": [ - { - "_count": "desc" - }, - { - "_key": "asc" - } - ] - } - } - } - } - } - }], - "triggers": [{ - "name": "DDOS-Trigger", - "severity": "1", - "condition": { - "script": { - "source": "ctx.results[0].aggregations.UDP2DstPortValueCount.value >= 1000 && ctx.results[0].aggregations.IPvDestAddrValueCount.value >= 1000 && ctx.results[0].aggregations.INTMetadataSourceMetadataOriginatingMacValueCount.value >= 1000", - "lang": "painless" - } - }, - "actions": [{ - "name": "Sdn-WebHook-Action", - "destination_id": "jjlCkXUBenaD-0rLO4Cg", - "message_template": { - "source":"{\"event_action\":\"trigger\",\"payload\":{ \"dst_ip\":\"{{_source.ts.IPv4DestIP}}\",\"dst_port\":\"{{_source.ts.UDP2DstPort}}\",\"src_ip\":\"{{_source.ts.IPv4SrcIP}}\",\"src_mac\":\"{{_source.ts.INTMetadataSourceMetadataOriginatingMac}}\"}}", - "lang": "mustache", - "options": { - "content_type": "application/json" - } - }, - "throttle_enabled": false - }] - }] -}' -``` -> **_NOTE:_** -> The message_template property is specific to each brought up instance in tofino build environment and changes accordingly. - -Once the associated monitor, trigger and action are configured you can test by sending packets specific to your testing scenario. diff --git a/analytics/OpenDistroElasticSearch/packet_template-7.x.sh b/analytics/OpenDistroElasticSearch/packet_template-7.x.sh deleted file mode 100755 index c6c373d4..00000000 --- a/analytics/OpenDistroElasticSearch/packet_template-7.x.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash - -if [[ $# -ne 1 ]] ; then - echo "packet_template.sh : Missing ElasticSearch server address .... " - exit -fi - -curl -H 'Content-Type: application/json' -XPUT 'http://'$1'/_template/packets' -d @data_mapping.json diff --git a/analytics/OpenDistroElasticSearch/udp_data_mapping_and_parsing_pipeline.md b/analytics/OpenDistroElasticSearch/udp_data_mapping_and_parsing_pipeline.md deleted file mode 100644 index 624adf15..00000000 --- a/analytics/OpenDistroElasticSearch/udp_data_mapping_and_parsing_pipeline.md +++ /dev/null @@ -1,34 +0,0 @@ -# UDP Data Mapping and Parsing Guide for Open Distro Elasticsearch - -- Default configured ElasticSearch instance runs on port 9200. -- Default configured Kibana instance runs on port 5601. - -## Adding UDP data templates to ElasticSearch for incoming packets -- To manually add UDP data templates for incoming UDP packets please run packet_template-7.x.sh script. The script takes ElasticSearch address and port number as first argument. -- Push it -``` - ./packet_template-7.x.sh localhost:9200 -``` -If you want to change the data templates for your packets, just revise the udp_data_parsing.json to your liking and re-run the packet_template-7.x.sh script.You will have to delete your existing packet indexing template, if any, before creating a new one. - -## Adding UDP data parsing to ElasticSearch for incoming packets -- To manually add UDP data parsing using Painless API for incoming UDP packets please run udp_data_parsing_pipeline.sh script. The script takes ElasticSearch address and port number as first argument. -- Push it -``` - ./udp_data_parsing_pipeline.sh localhost:9200 -``` - -If you want to change the data parsing pipeline for your packets, just revise the udp_data_parsing_pipeline.json to your liking and re-run the udp_data_parsing_pipeline.sh script. You will have to delete your existing parsing pipeline, if any, before creating a new one. - -## Commands to test added UDP templates and parsing pipeline for incoming packets -- To verify added UDP data template run the following ... - -``` - curl -H 'Content-Type: application/json' -XGET 'http://localhost:9200/_template/packets' -``` - -- To verify added UDP data parsing pipeline run the following ... - -``` - curl -H 'Content-Type: application/json' -XGET 'http://localhost:9200/_ingest/pipeline/ts_parsing' -``` diff --git a/analytics/OpenDistroElasticSearch/udp_data_parsing.json b/analytics/OpenDistroElasticSearch/udp_data_parsing.json deleted file mode 100644 index 6f1ba17a..00000000 --- a/analytics/OpenDistroElasticSearch/udp_data_parsing.json +++ /dev/null @@ -1,1246 +0,0 @@ -{ - "description": "Transparent Security UDP v4 & v6 data parsing pipeline", - "processors": [ - { - "set" : { - "field" : "ts.rawData", - "value" : "{{layers.data.data_data_data}}" - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": "ctx.ts.telemetryReport =ctx.ts.rawData.substring(0, 48).replace(':','')" - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String hardwareIdStr = ctx.layers.data.data_data_data.replace(':','').substring(1, 3); - BigInteger hwdIdBigInt = new BigInteger(hardwareIdStr, 16); - BigInteger bitwiseAnd = new BigInteger('FC', 16); - BigInteger extractedHardwareId = hwdIdBigInt.and(bitwiseAnd); - String hwdIdStr = extractedHardwareId.toString(16); - ctx.ts.telemetryHwdId = hwdIdStr; - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String sequenceNumberPartStrI = tsRawData.substring(2, 3); - String sequenceNumberPartStrII = tsRawData.substring(3, 8); - String sequenceNumberPartStr = tsRawData.substring(2, 8); - BigInteger sequenceNumberBigInt = new BigInteger(sequenceNumberPartStr, 16); - BigInteger bitwiseAnd = new BigInteger('7FFFFFF', 16); - BigInteger updatedSequenceNumber = sequenceNumberBigInt.and(bitwiseAnd); - // String sequenceNumberHexStr = updatedSequenceNumber.toString(16); - ctx.ts.telemetrySequenceNumber = updatedSequenceNumber.intValue(); - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String nodeIdStr = tsRawData.substring(8, 16); - BigInteger nodeIdBigInt = new BigInteger(nodeIdStr, 16); - ctx.ts.telemetryNodeId =nodeIdBigInt.intValue(); - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String repTypeStr = tsRawData.substring(16, 17); - BigInteger repTypeBigInt = new BigInteger(repTypeStr, 16); - ctx.ts.telemetryReportType =repTypeBigInt.intValue(); - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String inTypeStr = tsRawData.substring(17, 18); - BigInteger inTypeBigInt = new BigInteger(inTypeStr, 16); - ctx.ts.telemetryInType = inTypeBigInt.intValue(); - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String reportLengthStr = tsRawData.substring(18, 20); - BigInteger reportLengthBigInt = new BigInteger(reportLengthStr, 16); - ctx.ts.telemetryReportLength = reportLengthBigInt.intValue(); - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String mdLengthStr = tsRawData.substring(20, 22); - BigInteger mdLengthBigInt = new BigInteger(mdLengthStr, 16); - ctx.ts.telemetryMDLength = mdLengthBigInt.intValue(); - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String dStr = tsRawData.substring(22, 23); - BigInteger dBigInt = new BigInteger(dStr, 16); - BigInteger bitwiseAnd = new BigInteger('8', 16); - BigInteger dRes = dBigInt.and(bitwiseAnd); - ctx.ts.telemetryD = dRes.intValue(); - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String qStr = tsRawData.substring(22, 23); - BigInteger qBigInt = new BigInteger(qStr, 16); - BigInteger bitwiseAnd = new BigInteger('4', 16); - BigInteger qRes = qBigInt.and(bitwiseAnd); - ctx.ts.telemetryQ = qRes.intValue(); - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String fStr = tsRawData.substring(22, 23); - BigInteger fBigInt = new BigInteger(fStr, 16); - BigInteger bitwiseAnd = new BigInteger('2', 16); - BigInteger fRes = fBigInt.and(bitwiseAnd); - ctx.ts.telemetryF = fRes.intValue(); - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String IStr = tsRawData.substring(22, 23); - BigInteger IBigInt = new BigInteger(IStr, 16); - BigInteger bitwiseAnd = new BigInteger('1', 16); - BigInteger IRes = IBigInt.and(bitwiseAnd); - ctx.ts.telemetryI = IRes.intValue(); - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String rsvdStr = tsRawData.substring(23, 24); - BigInteger rsvdBigInt = new BigInteger(rsvdStr, 16); - ctx.ts.telemetryRsvd = rsvdBigInt.intValue(); - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String repMdBitsStr = tsRawData.substring(24, 28); - BigInteger repMdBitsBigInt = new BigInteger(repMdBitsStr, 16); - ctx.ts.telemetryRepMdBits = repMdBitsBigInt.intValue(); - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String domainSpecificIDStr = tsRawData.substring(28, 32); - BigInteger domainSpecificIDBigInt = new BigInteger(domainSpecificIDStr, 16); - ctx.ts.telemetryDomainSpecificID = domainSpecificIDBigInt.intValue(); - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String dsMdBitsStr = tsRawData.substring(32, 36); - BigInteger dsMdBitsStrBigInt = new BigInteger(dsMdBitsStr, 16); - ctx.ts.telemetryDsMdBits = dsMdBitsStrBigInt.intValue(); - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String dsMdStatusStr = tsRawData.substring(36, 40); - BigInteger dsMdStatusBigInt = new BigInteger(dsMdStatusStr, 16); - ctx.ts.telemetryDsMdStatus = dsMdStatusBigInt.intValue(); - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String varOptStr = tsRawData.substring(40, 48); - BigInteger varOptBigInt = new BigInteger(varOptStr, 16); - ctx.ts.telemetryVarOpt = varOptBigInt.intValue(); - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.isIPv6 = true; - } else - { - ctx.ts.isIPv6 = false; - } - - - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.isIPv4 = true; - } else - { - ctx.ts.isIPv4 = false; - } - - - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.data = tsRawData.substring(48, 252); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.data = tsRawData.substring(48, 292); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - ctx.ts.ethernetHeader = tsRawData.substring(48, 76); - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - ctx.ts.ethernetHeaderDestMac = tsRawData.substring(48, 60); - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - ctx.ts.ethernetHeaderSrcMac = tsRawData.substring(60, 72); - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - ctx.ts.ethernetHeaderEtherType = tsRawData.substring(72, 76); - - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.IPvData = tsRawData.substring(76, 116); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.IPvData = tsRawData.substring(76, 156); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.IPVersion = tsRawData.substring(76, 77); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.IPVersion = tsRawData.substring(76, 77); - } - """ - } - }, - { - "script" : { - "lang" : "painless", - "ignore_failure" : true, - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String IPv4SrcIPHexString = tsRawData.substring(100, 108); - String IPv4SrcIP = ""; - for(int i = 0; i < IPv4SrcIPHexString.length(); i = i + 2) { - IPv4SrcIP = IPv4SrcIP + Integer.valueOf(IPv4SrcIPHexString.substring(i, i+2), 16) + "."; - } - IPv4SrcIP = IPv4SrcIP.substring(0, IPv4SrcIP.length()-1); - ctx.ts.IPv4SrcIP = IPv4SrcIP; - } - if( etherType == '86dd' && ipVersion == '6') - { - String IPv6SrcIPHexString = tsRawData.substring(92,124); - String IPv6SrcIP = ""; - for(int i = 0; i < IPv6SrcIPHexString.length(); i = i + 2) { - IPv6SrcIP = IPv6SrcIP + Integer.valueOf(IPv6SrcIPHexString.substring(i, i+2), 16) + "."; - } - IPv6SrcIP = IPv6SrcIP.substring(0, IPv6SrcIP.length()-1); - ctx.ts.IPv6SrcIP = IPv6SrcIP; - } - """ - } - }, - { - "script" : { - "lang" : "painless", - "ignore_failure" : true, - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String IPv4DestIPHexString = tsRawData.substring(108, 116); - String IPv4DestIP = ""; - for(int i = 0; i < IPv4DestIPHexString.length(); i = i + 2) { - IPv4DestIP = IPv4DestIP + Integer.valueOf(IPv4DestIPHexString.substring(i, i+2), 16) + "."; - } - IPv4DestIP = IPv4DestIP.substring(0, IPv4DestIP.length()-1); - ctx.ts.IPv4DestIP = IPv4DestIP; - } - if( etherType == '86dd' && ipVersion == '6') - { - String IPv6DestIPHexString = tsRawData.substring(124,156); - String IPv6DestIP = ""; - for(int i = 0; i < IPv6DestIPHexString.length(); i = i + 2) { - IPv6DestIP = IPv6DestIP + Integer.valueOf(IPv6DestIPHexString.substring(i, i+2), 16) + "."; - } - IPv6DestIP = IPv6DestIP.substring(0, IPv6DestIP.length()-1); - ctx.ts.IPv6DestIP = IPv6DestIP; - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDGPData = tsRawData.substring(116, 132); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDGPData = tsRawData.substring(156, 172); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDGSrcPortStr = tsRawData.substring(116, 120); - BigInteger UDGSrcPortBigInt = new BigInteger(UDGSrcPortStr, 16); - ctx.ts.UDGSrcPort = UDGSrcPortBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDGSrcPortStr = tsRawData.substring(156, 160); - BigInteger UDGSrcPortBigInt = new BigInteger(UDGSrcPortStr, 16); - ctx.ts.UDGSrcPort = UDGSrcPortBigInt.intValue(); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDGDstPortStr = tsRawData.substring(120, 124); - BigInteger UDGDstPortBigInt = new BigInteger(UDGDstPortStr, 16); - ctx.ts.UDGDstPort = UDGDstPortBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDGDstPortStr = tsRawData.substring(160, 164); - BigInteger UDGDstPortBigInt = new BigInteger(UDGDstPortStr, 16); - ctx.ts.UDGDstPort = UDGDstPortBigInt.intValue(); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDGLengthStr = tsRawData.substring(124, 128); - BigInteger UDGLengthBigInt = new BigInteger(UDGLengthStr, 16); - ctx.ts.UDGLength = UDGLengthBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDGLengthStr = tsRawData.substring(164, 168); - BigInteger UDGLengthBigInt = new BigInteger(UDGLengthStr, 16); - ctx.ts.UDGLength = UDGLengthBigInt.intValue(); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDGChecksumStr = tsRawData.substring(128, 132); - BigInteger UDGChecksumBigInt = new BigInteger(UDGChecksumStr, 16); - ctx.ts.UDGChecksum = UDGChecksumBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDGChecksumStr = tsRawData.substring(168, 172); - BigInteger UDGChecksumBigInt = new BigInteger(UDGChecksumStr, 16); - ctx.ts.UDGChecksum = UDGChecksumBigInt.intValue(); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.InBandNetworkTelemetryData = tsRawData.substring(132, 196); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.InBandNetworkTelemetryData = tsRawData.substring(172, 236); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDPIntShimHeaderData = tsRawData.substring(132, 140); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDPIntShimHeaderData = tsRawData.substring(172, 180); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDPIntShimHeaderType = tsRawData.substring(132, 133); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDPIntShimHeaderType = tsRawData.substring(172, 173); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String udpIntShimNPTStr = tsRawData.substring(133, 134); - BigInteger udpIntShimNPTBigInt = new BigInteger(udpIntShimNPTStr, 16); - BigInteger bitwiseAnd = new BigInteger('C', 16); - BigInteger result = udpIntShimNPTBigInt.and(bitwiseAnd); - ctx.ts.UDPIntShimHeaderNPT = result.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String udpIntShimNPTStr = tsRawData.substring(173, 174); - BigInteger udpIntShimNPTBigInt = new BigInteger(udpIntShimNPTStr, 16); - BigInteger bitwiseAnd = new BigInteger('C', 16); - BigInteger result = udpIntShimNPTBigInt.and(bitwiseAnd); - ctx.ts.UDPIntShimHeaderNPT = result.intValue(); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDPIntShimHeaderRes1 = tsRawData.substring(133, 134); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDPIntShimHeaderRes1 = tsRawData.substring(174, 175); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String lengthStr = tsRawData.substring(134, 136); - BigInteger lengthStrBigInt = new BigInteger(lengthStr, 16); - ctx.ts.UDPIntShimHeaderLength= lengthStrBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String lengthStr = tsRawData.substring(174, 176); - BigInteger lengthStrBigInt = new BigInteger(lengthStr, 16); - ctx.ts.UDPIntShimHeaderLength = lengthStrBigInt.intValue(); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDPIntShimHeaderRes2 = tsRawData.substring(136, 138); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDPIntShimHeaderRes2 = tsRawData.substring(176, 178); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String nextProtoStr = tsRawData.substring(138, 140); - BigInteger nextProtoBigInt = new BigInteger(nextProtoStr, 16); - ctx.ts.UDPIntShimHeaderNextProto = nextProtoBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String nextProtoStr = tsRawData.substring(178, 180); - BigInteger nextProtoBigInt = new BigInteger(nextProtoStr, 16); - ctx.ts.UDPIntShimHeaderNextProto = nextProtoBigInt.intValue(); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.INTMetadataHeaderData = tsRawData.substring(140, 164); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.INTMetadataHeaderData = tsRawData.substring(180, 204); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderVersionStr = tsRawData.substring(140, 141); - BigInteger UDPIntMetaDataHeaderVersionStrBigInt = new BigInteger(UDPIntMetaDataHeaderVersionStr, 16); - ctx.ts.UDPIntMetaDataHeaderVersion = UDPIntMetaDataHeaderVersionStrBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderVersionStr = tsRawData.substring(180, 181); - BigInteger UDPIntMetaDataHeaderVersionStrBigInt = new BigInteger(UDPIntMetaDataHeaderVersionStr, 16); - ctx.ts.UDPIntMetaDataHeaderVersion = UDPIntMetaDataHeaderVersionStrBigInt.intValue(); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderRes = tsRawData.substring(141, 142); - BigInteger UDPIntMetaDataHeaderResBigInt = new BigInteger(UDPIntMetaDataHeaderRes, 16); - BigInteger bitwiseAnd = new BigInteger('C', 16); - BigInteger result = UDPIntMetaDataHeaderResBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderRes = result.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderRes = tsRawData.substring(181, 182); - BigInteger UDPIntMetaDataHeaderResBigInt = new BigInteger(UDPIntMetaDataHeaderRes, 16); - BigInteger bitwiseAnd = new BigInteger('C', 16); - BigInteger result = UDPIntMetaDataHeaderResBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderRes = result.intValue(); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderD = tsRawData.substring(141, 142); - BigInteger UDPIntMetaDataHeaderDBigInt = new BigInteger(UDPIntMetaDataHeaderD, 16); - BigInteger bitwiseAnd = new BigInteger('2', 16); - BigInteger result = UDPIntMetaDataHeaderDBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderD = result.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderD = tsRawData.substring(181, 182); - BigInteger UDPIntMetaDataHeaderDBigInt = new BigInteger(UDPIntMetaDataHeaderD, 16); - BigInteger bitwiseAnd = new BigInteger('2', 16); - BigInteger result = UDPIntMetaDataHeaderDBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderD = result.intValue(); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderE = tsRawData.substring(141, 142); - BigInteger UDPIntMetaDataHeaderEBigInt = new BigInteger(UDPIntMetaDataHeaderE, 16); - BigInteger bitwiseAnd = new BigInteger('1', 16); - BigInteger result = UDPIntMetaDataHeaderEBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderE = result.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderE = tsRawData.substring(181, 182); - BigInteger UDPIntMetaDataHeaderEBigInt = new BigInteger(UDPIntMetaDataHeaderE, 16); - BigInteger bitwiseAnd = new BigInteger('1', 16); - BigInteger result = UDPIntMetaDataHeaderEBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderE = result.intValue(); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderM = tsRawData.substring(142, 143); - BigInteger UDPIntMetaDataHeaderMBigInt = new BigInteger(UDPIntMetaDataHeaderM, 16); - BigInteger bitwiseAnd = new BigInteger('8', 16); - BigInteger result = UDPIntMetaDataHeaderMBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderM = result.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderM = tsRawData.substring(182, 183); - BigInteger UDPIntMetaDataHeaderMBigInt = new BigInteger(UDPIntMetaDataHeaderM, 16); - BigInteger bitwiseAnd = new BigInteger('8', 16); - BigInteger result = UDPIntMetaDataHeaderMBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderM = result.intValue(); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderPerhopMetadataLengthStr = tsRawData.substring(144, 146); - BigInteger UDPIntMetaDataHeaderPerhopMetadataLengthBigInt = new BigInteger(UDPIntMetaDataHeaderPerhopMetadataLengthStr, 16); - BigInteger bitwiseAnd = new BigInteger('1F', 16); - BigInteger result = UDPIntMetaDataHeaderPerhopMetadataLengthBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderPerhopMetadataLength = result.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderPerhopMetadataLengthStr = tsRawData.substring(184, 186); - BigInteger UDPIntMetaDataHeaderPerhopMetadataLengthBigInt = new BigInteger(UDPIntMetaDataHeaderPerhopMetadataLengthStr, 16); - BigInteger bitwiseAnd = new BigInteger('1F', 16); - BigInteger result = UDPIntMetaDataHeaderPerhopMetadataLengthBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderPerhopMetadataLength = result.intValue(); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderRemainingHopCntStr = tsRawData.substring(146, 148); - BigInteger UDPIntMetaDataHeaderRemainingHopCntBigInt = new BigInteger(UDPIntMetaDataHeaderRemainingHopCntStr, 16); - ctx.ts.UDPIntMetaDataHeaderRemainingHopCnt = UDPIntMetaDataHeaderRemainingHopCntBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderRemainingHopCntStr = tsRawData.substring(186, 188); - BigInteger UDPIntMetaDataHeaderRemainingHopCntBigInt = new BigInteger(UDPIntMetaDataHeaderRemainingHopCntStr, 16); - ctx.ts.UDPIntMetaDataHeaderRemainingHopCnt = UDPIntMetaDataHeaderRemainingHopCntBigInt.intValue(); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderInstructionBitmapStr = tsRawData.substring(148, 152); - BigInteger UDPIntMetaDataHeaderInstructionBitmapBigInt = new BigInteger(UDPIntMetaDataHeaderInstructionBitmapStr, 16); - ctx.ts.UDPIntMetaDataHeaderInstructionBitmap = UDPIntMetaDataHeaderInstructionBitmapBigInt.toString(16); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderInstructionBitmapStr = tsRawData.substring(188, 192); - BigInteger UDPIntMetaDataHeaderInstructionBitmapBigInt = new BigInteger(UDPIntMetaDataHeaderInstructionBitmapStr, 16); - ctx.ts.UDPIntMetaDataHeaderInstructionBitmap = UDPIntMetaDataHeaderInstructionBitmapBigInt.toString(16); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderDomainSpecificIDStr = tsRawData.substring(152, 156); - BigInteger UDPIntMetaDataHeaderDomainSpecificIDBigInt = new BigInteger(UDPIntMetaDataHeaderDomainSpecificIDStr, 16); - ctx.ts.UDPIntMetaDataHeaderDomainSpecificID = UDPIntMetaDataHeaderDomainSpecificIDBigInt.toString(16); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderDomainSpecificIDStr = tsRawData.substring(192, 196); - BigInteger UDPIntMetaDataHeaderDomainSpecificIDBigInt = new BigInteger(UDPIntMetaDataHeaderDomainSpecificIDStr, 16); - ctx.ts.UDPIntMetaDataHeaderDomainSpecificID = UDPIntMetaDataHeaderDomainSpecificIDBigInt.toString(16); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderDSInstructionStr = tsRawData.substring(156, 160); - BigInteger UDPIntMetaDataHeaderDSInstructionBigInt = new BigInteger(UDPIntMetaDataHeaderDSInstructionStr, 16); - ctx.ts.UDPIntMetaDataHeaderDSInstruction = UDPIntMetaDataHeaderDSInstructionBigInt.toString(16); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderDSInstructionStr = tsRawData.substring(196, 200); - BigInteger UDPIntMetaDataHeaderDSInstructionBigInt = new BigInteger(UDPIntMetaDataHeaderDSInstructionStr, 16); - ctx.ts.UDPIntMetaDataHeaderDSInstruction = UDPIntMetaDataHeaderDSInstructionBigInt.toString(16); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderDSFlagsStr = tsRawData.substring(160, 164); - BigInteger UDPIntMetaDataHeaderDSFlagsBigInt = new BigInteger(UDPIntMetaDataHeaderDSFlagsStr, 16); - ctx.ts.UDPIntMetaDataHeaderDSFlags = UDPIntMetaDataHeaderDSFlagsBigInt.toString(16); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderDSFlagsStr = tsRawData.substring(200, 204); - BigInteger UDPIntMetaDataHeaderDSFlagsBigInt = new BigInteger(UDPIntMetaDataHeaderDSFlagsStr, 16); - ctx.ts.UDPIntMetaDataHeaderDSFlags = UDPIntMetaDataHeaderDSFlagsBigInt.toString(16); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.INTMetadataStackData = tsRawData.substring(164, 196); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.INTMetadataStackData = tsRawData.substring(204, 236); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.INTMetadataStackSwitchID = tsRawData.substring(164, 172); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.INTMetadataStackSwitchID = tsRawData.substring(204, 212); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.INTMetadataSourceMetadata = tsRawData.substring(172, 196); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.INTMetadataSourceMetadata = tsRawData.substring(212, 236); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.INTMetadataSourceMetadataSwitchId = tsRawData.substring(172, 180); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.INTMetadataSourceMetadataSwitchId = tsRawData.substring(212, 220); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.INTMetadataSourceMetadataOriginatingMac = tsRawData.substring(180, 192); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.INTMetadataSourceMetadataOriginatingMac = tsRawData.substring(220, 232); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.INTMetadataSourceMetadataReserved = tsRawData.substring(192, 196); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.INTMetadataSourceMetadataReserved = tsRawData.substring(232, 236); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDP2Data = tsRawData.substring(196, 212); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDP2Data = tsRawData.substring(236, 252); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDP2SrcPortStr = tsRawData.substring(196, 200); - BigInteger UDP2SrcPortBigInt = new BigInteger(UDP2SrcPortStr,16); - ctx.ts.UDP2SrcPort = UDP2SrcPortBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDP2SrcPortStr = tsRawData.substring(236, 240); - BigInteger UDP2SrcPortBigInt = new BigInteger(UDP2SrcPortStr,16); - ctx.ts.UDP2SrcPort = UDP2SrcPortBigInt.intValue(); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDP2DstPortStr = tsRawData.substring(200, 204); - BigInteger UDP2DstPortBigInt = new BigInteger(UDP2DstPortStr,16); - ctx.ts.UDP2DstPort = UDP2DstPortBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDP2DstPortStr = tsRawData.substring(240, 244); - BigInteger UDP2DstPortBigInt = new BigInteger(UDP2DstPortStr,16); - ctx.ts.UDP2DstPort = UDP2DstPortBigInt.intValue(); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDP2Length = tsRawData.substring(204, 208); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDP2Length = tsRawData.substring(244, 248); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDP2Checksum = tsRawData.substring(208, 212); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDP2Checksum = tsRawData.substring(248, 252); - } - """ - } - }, - { - "script": { - "lang": "painless", - "ignore_failure": true, - "source": """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.dataII = tsRawData.substring(212, 252); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.dataII = tsRawData.substring(252, 292); - } - """ - } - } - ] -} diff --git a/analytics/OpenDistroElasticSearch/udp_data_parsing_pipeline.sh b/analytics/OpenDistroElasticSearch/udp_data_parsing_pipeline.sh deleted file mode 100755 index 24cc9652..00000000 --- a/analytics/OpenDistroElasticSearch/udp_data_parsing_pipeline.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash - -if [[ $# -ne 1 ]] ; then - echo "udp_data_parsing.sh : Missing ElasticSearch server address .... " - exit -fi - -curl -H 'Content-Type: application/json' -XPUT 'http://'$1'/_ingest/pipeline/ts_parsing' -d @udp_data_parsing.json diff --git a/analytics/Readme.md b/analytics/Readme.md deleted file mode 100644 index 96d62e14..00000000 --- a/analytics/Readme.md +++ /dev/null @@ -1,76 +0,0 @@ -# Analytics Guide - -The transparent security analytics module uses the following components for data capture and data analysis. - -1. Open source python program Espcap for live data capture. ( https://github.com/vichargrave/espcap ) -2. Open source distribution of ElasticSearch. ( https://opendistro.github.io/for-elasticsearch-docs/ ) - -## Requirements -**Espcap** relies on the Elasticsearch Python Client module to index packets in Elasticsearch. The version of the client module must match the version of Elasticsearch you want to use. -### Support for *Elasticsearch 7.x* requires: - -1. Python 3.7 (Python 2.7.x not supported) -2. TShark 3.0.1 (included in Wireshark) -3. *Click* module for Python -4. Elasticsearch Python Client module 7.x -5. Elasticsearch 7.x - -### Support for *Elasticsearch 6.x* requires: - -1. Requirements 1 - 3 listed above -2. Elasticsearch Python Client module 6.x -3. Elasticsearch 6.x - -## Installation -- The current setup in transparent security analytics tofino build environment is configured with an CENTOS AMI which has all the pre-requisites installed. -- To manually setup your CENTOS machine please follow the steps from [INSTALL](setup/INSTALL.md) Guide. - -## Running Examples - -- To start a live capture on an analytics instance from the network interface `ae-eth0`, get all packets and index them in the Elasticsearch cluster running at localhost:9200 , assuming your present working directory is *espcap* - ``` - cd espcap/ - sudo python3 src/espcap.py --nic=ae-eth0 --node=localhost:9200 --chunk=100 - ``` -The packets get captured and indexed in ElasticSearch under packets-* index. - -- Display the following help message: - ``` - cd espcap/ - espcap.py --help - Usage: espcap.py [OPTIONS] - - Options: - --node TEXT Elasticsearch IP and port (default=None, dump packets to - stdout) - --nic TEXT Network interface for live capture (default=None, if file - or dir specified) - --file TEXT PCAP file for file capture (default=None, if nic specified) - --dir TEXT PCAP directory for multiple file capture (default=None, if - nic specified) - --bpf TEXT Packet filter for live capture (default=all packets) - --chunk INTEGER Number of packets to bulk index (default=1000) - --count INTEGER Number of packets to capture during live capture - (default=0, capture indefinitely) - --list List the network interfaces - --help Show this message and exit. - ``` - -## Packet Indexing -Default installation of ElasticSearch cluster is at port 9200 and default installation of Kibana runs at port 5601. - -To visualize the indexed packets / incoming data in Kibana you need to create an Index pattern for incoming packets. This is a manual step. Do the following steps for the same. - - 1. To access Kibana UI go localhost:5601 => Stack Management => Index Patters => Create index pattern. - 2. Add a name for Index pattern and select the corresponding matching index source. - 3. Click on next step and choose timestamp as primary time field for the index pattern. - 4. Click create index pattern and navigate to Discover tab to see the incoming packets matching the index pattern. - - - - - - - - - diff --git a/analytics/setup/INSTALL.md b/analytics/setup/INSTALL.md deleted file mode 100644 index 6ad6af7b..00000000 --- a/analytics/setup/INSTALL.md +++ /dev/null @@ -1 +0,0 @@ -# TS Analytics environment Setup Guide for CentOS diff --git a/automation/env-build/aws-inst.tf b/automation/env-build/aws-inst.tf index a7422952..32dc1311 100644 --- a/automation/env-build/aws-inst.tf +++ b/automation/env-build/aws-inst.tf @@ -22,17 +22,15 @@ locals { ] py2 = [ "sudo apt-get update", - "sudo apt-get install python2.7 aptitude -y", - ] - ae = [ - "sudo rm -f /usr/bin/python", - "sudo ln -s /usr/bin/python2.7 /usr/bin/python" + "sudo apt-get install python2.7 -y", + "sudo apt-get update", + "sudo apt-get install aptitude -y", ] - inline_scripts = var.env_type == "ae" ? local.ae : var.env_type == "tofino" ? local.py2 : var.env_type == "siddhi" ? local.none : local.py3 + inline_scripts = var.env_type == "tofino" ? local.py2 : local.none } resource "aws_instance" "transparent-security-build-img" { - ami = var.env_type == "ae" ? var.centos7_ami : var.ubuntu_version == "20" ? var.base_20_ami : var.ubuntu_version == "18" ? var.base_18_ami : var.base_16_ami + ami = var.ubuntu_version == "20" ? var.base_20_ami : var.ubuntu_version == "18" ? var.base_18_ami : var.base_16_ami instance_type = var.instance_type key_name = aws_key_pair.transparent-security-mini-pk.key_name @@ -75,14 +73,9 @@ ${var.ANSIBLE_PB_PATH}/env-build/${var.env_type}/env_build.yml \ --extra-vars "\ aws_access_key=${var.access_key} aws_secret_key=${var.secret_key} -grpc_version=${var.grpc_version} -p4c_version=${var.p4c_version} -protobuf_version=${var.protobuf_version} -pi_version=${var.pi_version} -bm_version=${var.bm_version} bf_sde_version=${var.bf_sde_version} bf_sde_profile=${var.bf_sde_profile} -bf_sde_s3_bucket=${var.bf_sde_s3_bucket} +bf_sde_s3_bucket=${var.bf_sde_s3_bucket == null ? "n/a" : var.bf_sde_s3_bucket} remote_scripts_dir=${var.remote_scripts_dir} python_version=${var.python_version} ubuntu_version=${var.ubuntu_version} diff --git a/automation/env-build/variables.tf b/automation/env-build/variables.tf index 536cad00..bcd9b88f 100644 --- a/automation/env-build/variables.tf +++ b/automation/env-build/variables.tf @@ -16,15 +16,15 @@ variable "access_key" {} variable "secret_key" {} variable "build_id" {} variable "ec2_region" {} -variable "env_type" {default = "mininet"} +variable "env_type" {default = "tofino"} # Dependency version only for tofino environments -variable "bf_sde_s3_bucket" {default = "null"} +variable "bf_sde_s3_bucket" {default = null} # Optional Variables variable "bf_sde_version" {default = "9.2.0"} -variable "bf_sde_profile" {default = "p4_runtime_profile"} -variable "create_ami" {default = "yes"} +variable "bf_sde_profile" {default = "p416_examples_profile"} +variable "create_ami" {default = "no"} variable "public_key_file" {default = "~/.ssh/id_rsa.pub"} variable "private_key_file" {default = "~/.ssh/id_rsa"} variable "sudo_user" {default = "ubuntu"} @@ -32,9 +32,6 @@ variable "ubuntu_version" {default = "18"} variable "siddhi_map_p4_trpt_version" {default = "master"} variable "python_version" {default = "3.6"} -# snaps-hcp image -variable "centos7_ami" {default="ami-01ed306a12b7d1c96"} - # ubuntu 16 variable "base_16_ami" {default = "ami-08692d171e3cf02d6"} # ubuntu 18 @@ -45,12 +42,6 @@ variable "base_20_ami" {default = "ami-03d5c68bab01f3496"} variable "instance_type" {default = "t2.2xlarge"} variable "run_tests" {default = "yes"} -# Dependency versions only for mininet environments -variable "grpc_version" {default = "v1.19.1"} -variable "p4c_version" {default = "fbe395bbf1eed9653323ac73b20cf6c06af2121e"} -variable "protobuf_version" {default = "3.6.x"} -variable "pi_version" {default = "1539ecd8a50c159b011d9c5a9c0eba99f122a845"} -variable "bm_version" {default = "16c699953ee02306731ebf9a9241ea9fe3bbdc8c"} variable "remote_scripts_dir" {default = "/etc/transparent-security"} # Playbook Constants diff --git a/automation/p4/mininet/aws-inst.tf b/automation/p4/mininet/aws-inst.tf deleted file mode 100644 index 77412b6f..00000000 --- a/automation/p4/mininet/aws-inst.tf +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# AWS EC2 Instance -resource "aws_instance" "transparent-security-mininet-integration" { - ami = var.mininet_ami - instance_type = var.instance_type - key_name = aws_key_pair.transparent-security-pk.key_name - - tags = { - Name = "transparent-security-transparent-security-${var.scenario_name}-${var.build_id}" - } - - security_groups = [aws_security_group.transparent-security-img-sg.name] - associate_public_ip_address = true - - # Used to ensure host is really up before attempting to apply ansible playbooks - provisioner "remote-exec" { - inline = [ - "sudo echo 'transparent-security mininet integration CI' > ~/motd", - ] - } - - # Remote connection info for remote-exec - connection { - host = self.public_ip - type = "ssh" - user = var.sudo_user - private_key = file(var.private_key_file) - } - - root_block_device { - volume_size = "50" - } -} - -resource "aws_instance" "transparent-security-hcp-instance" { - count = var.scenario_name == "lab_trial" ? 1 : 0 - ami = var.hcp_ami - instance_type = var.instance_type - key_name = aws_key_pair.transparent-security-pk.key_name - - tags = { - Name = "transparent-security-hcp-${var.scenario_name}-${var.build_id}" - } - - security_groups = [aws_security_group.transparent-security-img-sg.name, aws_security_group.transparent-security-hcp-img-sg.name ] - associate_public_ip_address = true - - # Used to ensure host is really up before attempting to apply ansible playbooks - provisioner "remote-exec" { - inline = [ - "sudo echo 'transparent-security-hcp integration' > ~/motd", - ] - } - - # Remote connection info for remote-exec - connection { - host = self.public_ip - type = "ssh" - user = var.hcp_sudo_user - private_key = file(var.private_key_file) - } -} diff --git a/automation/p4/mininet/aws-security.tf b/automation/p4/mininet/aws-security.tf deleted file mode 100644 index c3d82009..00000000 --- a/automation/p4/mininet/aws-security.tf +++ /dev/null @@ -1,89 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# AWS Credentials -provider "aws" { - access_key = var.access_key - secret_key = var.secret_key - region = var.ec2_region -} - -# Note: Script will fail if another process is leveraging the same build_id -resource "aws_security_group" "transparent-security-img-sg" { - name = "transparent-security-${var.scenario_name}-${var.build_id}" - ingress { - cidr_blocks = ["0.0.0.0/0"] - from_port = 22 - to_port = 22 - protocol = "tcp" - } - - ingress { - cidr_blocks = ["0.0.0.0/0"] - from_port = var.sdn_port - to_port = var.sdn_port - protocol = "tcp" - } - - ingress { - cidr_blocks = ["0.0.0.0/0"] - from_port = 50051 - to_port = 50056 - protocol = "tcp" - } - - // Terraform removes the default rule - egress { - from_port = 0 - to_port = 0 - protocol = "-1" - cidr_blocks = ["0.0.0.0/0"] - } -} - -resource "aws_security_group" "transparent-security-hcp-img-sg" { - name = "transparent-security-hcp-${var.scenario_name}-${var.build_id}" - ingress { - cidr_blocks = ["0.0.0.0/0"] - from_port = 22 - to_port = 22 - protocol = "tcp" - } - - ingress { - cidr_blocks = ["0.0.0.0/0"] - from_port = var.sdn_port - to_port = var.sdn_port - protocol = "tcp" - } - - ingress { - cidr_blocks = ["0.0.0.0/0"] - from_port = 50051 - to_port = 50056 - protocol = "tcp" - } - - // Terraform removes the default rule - egress { - from_port = 0 - to_port = 0 - protocol = "-1" - cidr_blocks = ["0.0.0.0/0"] - } -} - -# AWS EC2 Instance Public Key -resource "aws_key_pair" "transparent-security-pk" { - public_key = file(var.public_key_file) -} diff --git a/automation/p4/mininet/outputs.tf b/automation/p4/mininet/outputs.tf deleted file mode 100644 index acf5757f..00000000 --- a/automation/p4/mininet/outputs.tf +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Outputs -output "ip" { - value = aws_instance.transparent-security-mininet-integration.public_ip -} - -output "hcp-ip" { - value = var.scenario_name == "lab_trial" ? aws_instance.transparent-security-hcp-instance.0.public_ip : "n/a" -} - -output "hcp-private-ip" { - value = var.scenario_name == "lab_trial" ? aws_instance.transparent-security-hcp-instance.0.private_ip: "n/a" -} diff --git a/automation/p4/mininet/scenarios.tf b/automation/p4/mininet/scenarios.tf deleted file mode 100644 index d45c7f62..00000000 --- a/automation/p4/mininet/scenarios.tf +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -resource "null_resource" "transparent-security-run-scenario-tests" { - depends_on = [ - null_resource.transparent-security-start-sim, - null_resource.transparent-security-hcp-tunnel, - ] - - provisioner "remote-exec" { - inline = [ - "sudo pip install ansible", - "${var.ANSIBLE_CMD} -i ${var.remote_inventory_file} ${var.remote_scenario_pb_dir}/${var.scenario_name}/${var.test_case}.yml", - ] - } - - connection { - host = aws_instance.transparent-security-mininet-integration.public_ip - type = "ssh" - user = var.sudo_user - private_key = file(var.private_key_file) - } -} diff --git a/automation/p4/mininet/setup.tf b/automation/p4/mininet/setup.tf deleted file mode 100644 index 27e1b48b..00000000 --- a/automation/p4/mininet/setup.tf +++ /dev/null @@ -1,150 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Create and inject own SSH keys for being able to SSH into self or mininet hosts -resource "null_resource" "transparent-security-host-ssh-setup" { - provisioner "remote-exec" { - inline = [ - "ssh-keygen -t rsa -N '' -f ~/.ssh/id_rsa", - "touch ~/.ssh/authorized_keys", - "chmod 600 ~/.ssh/authorized_keys", - "cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys", - ] - } - connection { - host = aws_instance.transparent-security-mininet-integration.public_ip - type = "ssh" - user = var.sudo_user - private_key = file(var.private_key_file) - } -} - -locals { - local_ansible_inventory_file = "~/tps-mininet-setup-${var.scenario_name}-${var.build_id}.ini" -} - -#Create a local inventory to store variables and public IP of remote machine -resource "null_resource" "transparent-security-local-inventory" { - depends_on = [null_resource.transparent-security-host-ssh-setup] - provisioner "local-exec" { - command = <-.yml -variable "test_case" {default = "all"} diff --git a/automation/p4/tofino/aws-insts.tf b/automation/p4/tofino/aws-insts.tf index 4e39b7b2..bf53efb6 100644 --- a/automation/p4/tofino/aws-insts.tf +++ b/automation/p4/tofino/aws-insts.tf @@ -12,7 +12,11 @@ # limitations under the License. locals { - ami = var.p4_arch == "tna" ? var.tofino.bfrt_ami : var.tofino.p4rt_ami + ami = var.tofino.bfrt_ami + dflt_start_script = ["sudo echo 'tps-orchestrator/controller' > ~/motd"] + // Placeholder for custom startup options for BFRT images + tofino_img_start_script = local.dflt_start_script + start_script = var.p4_arch == "tna" ? local.tofino_img_start_script : local.dflt_start_script } # Orchestrator/SDN Controller Instance @@ -29,9 +33,7 @@ resource "aws_instance" "orchestrator" { associate_public_ip_address = true provisioner "remote-exec" { - inline = [ - "sudo echo 'tps-orchestrator/controller' > ~/motd", - ] + inline = local.start_script } # Remote connection info for remote-exec @@ -45,8 +47,8 @@ resource "aws_instance" "orchestrator" { # Determine switch and node count locals { - switch_count = var.scenario_name == "full" ? var.num_switches_full : var.scenario_name == "lab_trial" ? var.num_switches_lab_trial : var.num_switches_single - node_count = var.scenario_name == "full" ? var.num_nodes_full : var.scenario_name == "lab_trial" ? var.num_nodes_lab_trial : var.num_nodes_single + switch_count = var.scenario_name == "lab_trial" ? var.num_switches_lab_trial : var.num_switches_single + node_count = var.scenario_name == "lab_trial" ? var.num_nodes_lab_trial : var.num_nodes_single } # Tofino Model Switch Instances @@ -63,6 +65,18 @@ resource "aws_instance" "tps-switch" { security_groups = [aws_security_group.tps.name] associate_public_ip_address = true + + provisioner "remote-exec" { + inline = local.start_script + } + + # Remote connection info for remote-exec + connection { + host = self.public_ip + type = "ssh" + user = var.switch_user + private_key = file(var.private_key_file) + } } # Third octet of the subnet IPv4 value diff --git a/automation/p4/tofino/setup.tf b/automation/p4/tofino/setup.tf index 165d6515..e7663cb9 100644 --- a/automation/p4/tofino/setup.tf +++ b/automation/p4/tofino/setup.tf @@ -14,46 +14,33 @@ locals { sdn_ip = aws_instance.orchestrator.private_ip - # For full & lab_trial scenarios - core_switch_ip = var.scenario_name == "full" || var.scenario_name == "lab_trial" ? aws_instance.tps-switch.0.private_ip: "n/a" - core_tun1_ip = var.scenario_name == "full" || var.scenario_name == "lab_trial" ? aws_network_interface.switch_tun_1.0.private_ip: "n/a" - core_tun1_mac = var.scenario_name == "full" || var.scenario_name == "lab_trial" ? aws_network_interface.switch_tun_1.0.mac_address: "n/a" - agg_switch_ip = var.scenario_name == "full" || var.scenario_name == "lab_trial" ? aws_instance.tps-switch.1.private_ip: "n/a" - agg_tun1_ip = var.scenario_name == "full" || var.scenario_name == "lab_trial" ? aws_network_interface.switch_tun_1.1.private_ip: "n/a" - agg_tun1_mac = var.scenario_name == "full" || var.scenario_name == "lab_trial" ? aws_network_interface.switch_tun_1.1.mac_address: "n/a" - - # For full scenario - gateway_1_ip = var.scenario_name == "full" ? aws_instance.tps-switch.2.private_ip: "n/a" - gateway_2_ip = var.scenario_name == "full" ? aws_instance.tps-switch.3.private_ip: "n/a" - gateway_3_ip = var.scenario_name == "full" ? aws_instance.tps-switch.4.private_ip: "n/a" - camera_1_ip = var.scenario_name == "full" ? aws_instance.node.0.private_ip: "n/a" - nas_1_ip = var.scenario_name == "full" ? aws_instance.node.1.private_ip: "n/a" - game_1_ip = var.scenario_name == "full" ? aws_instance.node.2.private_ip: "n/a" - camera_2_ip = var.scenario_name == "full" ? aws_instance.node.3.private_ip: "n/a" - game_2_ip = var.scenario_name == "full" ? aws_instance.node.4.private_ip: "n/a" - camera_3_ip = var.scenario_name == "full" ? aws_instance.node.5.private_ip: "n/a" - game_3_ip = var.scenario_name == "full" ? aws_instance.node.6.private_ip: "n/a" - # For lab_trial scenario - ae_ip = var.scenario_name == "lab_trial" ? aws_instance.ae.private_ip: "n/a" + core_switch_ip = var.scenario_name == "lab_trial" ? aws_instance.tps-switch.0.private_ip: "n/a" + core_tun1_ip = var.scenario_name == "lab_trial" ? aws_network_interface.switch_tun_1.0.private_ip: "n/a" + core_tun1_mac = var.scenario_name == "lab_trial" ? aws_network_interface.switch_tun_1.0.mac_address: "n/a" + agg_switch_ip = var.scenario_name == "lab_trial" ? aws_instance.tps-switch.1.private_ip: "n/a" + agg_tun1_ip = var.scenario_name == "lab_trial" ? aws_network_interface.switch_tun_1.1.private_ip: "n/a" + agg_tun1_mac = var.scenario_name == "lab_trial" ? aws_network_interface.switch_tun_1.1.mac_address: "n/a" + ae_ip = var.scenario_name == "lab_trial" ? var.ae_k8s_svc_ip: "n/a" + ae_mgmt_ip = var.scenario_name == "lab_trial" ? aws_instance.ae.private_ip: "n/a" ae_tun1_ip = var.scenario_name == "lab_trial" ? aws_network_interface.ae_tun_1.private_ip: "n/a" ae_tun1_mac = var.scenario_name == "lab_trial" ? aws_network_interface.ae_tun_1.mac_address: "n/a" # For single-switch scenario - switch_ip = var.scenario_name == "full" ? "n/a" : aws_instance.tps-switch.0.private_ip - switch_tun1_ip = var.scenario_name == "full" ? "n/a" : aws_network_interface.switch_tun_1.0.private_ip - switch_tun1_mac = var.scenario_name == "full" ? "n/a" : aws_network_interface.switch_tun_1.0.mac_address - clone_ip = var.scenario_name == "full" ? "n/a" : aws_instance.node.2.private_ip - clone_tun1_ip = var.scenario_name == "full" ? "n/a" : aws_network_interface.node_tun_1.2.private_ip - clone_tun1_mac = var.scenario_name == "full" ? "n/a" : aws_network_interface.node_tun_1.2.mac_address + switch_ip = aws_instance.tps-switch.0.private_ip + switch_tun1_ip = aws_network_interface.switch_tun_1.0.private_ip + switch_tun1_mac = aws_network_interface.switch_tun_1.0.mac_address + clone_ip = aws_instance.node.2.private_ip + clone_tun1_ip = aws_network_interface.node_tun_1.2.private_ip + clone_tun1_mac = aws_network_interface.node_tun_1.2.mac_address # For single_switch & lab_trial scenarios - host1_ip = var.scenario_name == "full" ? "n/a" : aws_instance.node.0.private_ip - host1_tun1_ip = var.scenario_name == "full" ? "n/a" : aws_network_interface.node_tun_1.0.private_ip - host1_tun1_mac = var.scenario_name == "full" ? "n/a" : aws_network_interface.node_tun_1.0.mac_address - host2_ip = var.scenario_name == "full" ? "n/a" : aws_instance.node.1.private_ip - host2_tun1_ip = var.scenario_name == "full" ? "n/a" : aws_network_interface.node_tun_1.1.private_ip - host2_tun1_mac = var.scenario_name == "full" ? "n/a" : aws_network_interface.node_tun_1.1.mac_address + host1_ip = aws_instance.node.0.private_ip + host1_tun1_ip = aws_network_interface.node_tun_1.0.private_ip + host1_tun1_mac = aws_network_interface.node_tun_1.0.mac_address + host2_ip = aws_instance.node.1.private_ip + host2_tun1_ip = aws_network_interface.node_tun_1.1.private_ip + host2_tun1_mac = aws_network_interface.node_tun_1.1.mac_address # For lab_trial scenarios lab_inet_ip = var.scenario_name == "lab_trial" ? aws_instance.node.2.private_ip: "n/a" @@ -92,7 +79,7 @@ EOT } resource "null_resource" "tps-sim-setup-orch-single-switch" { - count = var.scenario_name == "full" || var.scenario_name == "lab_trial" ? 0 : 1 + count = var.scenario_name == "lab_trial" ? 0 : 1 depends_on = [ aws_instance.tps-switch, aws_instance.node, @@ -165,6 +152,7 @@ inet_ip=${local.lab_inet_ip} inet_tun1_ip=${local.lab_inet_tun1_ip} inet_tun1_mac=${local.lab_inet_tun1_mac} ae_ip=${local.ae_ip} +ae_mgmt_ip=${local.ae_mgmt_ip} ae_tun1_ip=${local.ae_tun1_ip} ae_tun1_mac=${local.ae_tun1_mac} switch_user=${var.switch_user} @@ -192,7 +180,7 @@ EOT } locals { - setup_pb = var.scenario_name == "full" || var.scenario_name == "lab_trial" ? "setup_nodes-${var.scenario_name}.yml" : "setup_nodes-single_switch.yml" + setup_pb = var.scenario_name == "lab_trial" ? "setup_nodes-${var.scenario_name}.yml" : "setup_nodes-single_switch.yml" } resource "null_resource" "tps-tofino-setup-nodes" { diff --git a/automation/p4/tofino/variables.tf b/automation/p4/tofino/variables.tf index 6d6472e5..478ad348 100644 --- a/automation/p4/tofino/variables.tf +++ b/automation/p4/tofino/variables.tf @@ -31,27 +31,23 @@ variable "availability_zone" {default = "us-west-2a"} variable "tofino" { default = { sde_version = "9.2.0" - p4rt_ami = "ami-043e6714f3d0863f2" - bfrt_ami = "ami-01a5ff54de23b6739" + bfrt_ami = "ami-090b409398f016b84" } } -variable "siddhi_ae_ami" {default = "ami-00876f36fe8f97733"} +variable "siddhi_ae_ami" {default = "ami-010458ceb325ec348"} variable "switch_instance_type" {default = "t2.2xlarge"} variable "ae_instance_type" {default = "t2.2xlarge"} variable "orch_instance_type" {default = "t2.2xlarge"} variable "node_instance_type" {default = "t2.small"} -variable "num_switches_full" {default = 5} variable "num_switches_single" {default = 1} variable "num_switches_lab_trial" {default = 2} -variable "num_nodes_full" {default = 9} variable "num_nodes_single" {default = 3} variable "num_nodes_lab_trial" {default = 3} # Variables for ansible playbooks variable "ANSIBLE_CMD" {default = "export ANSIBLE_HOST_KEY_CHECKING=False; ansible-playbook"} -variable "SETUP_ORCH_FULL" {default = "../../../playbooks/tofino/setup_orchestrator-full.yml"} variable "SETUP_ORCH_SINGLE_SWITCH" {default = "../../../playbooks/tofino/setup_orchestrator-single_switch.yml"} variable "SETUP_ORCH_LAB_TRIAL" {default = "../../../playbooks/tofino/setup_orchestrator-lab_trial.yml"} variable "START_SERVICE" {default = "../../../playbooks/general/start_service.yml"} @@ -70,6 +66,7 @@ variable "p4_grpc_port" {default = "50051"} variable "bf_grpc_port" {default = "50052"} variable "p4_bridge_subnet" {default = "192.168.0.0/24"} variable "sdn_port" {default = "9998"} +variable "ae_k8s_svc_ip" {default = "10.96.0.2"} variable "switch_nic_prfx" {default = "veth"} variable "service_log_level" {default = "DEBUG"} variable "ae_monitor_intf" {default = "core-tun"} @@ -79,5 +76,5 @@ variable "p4_arch" {default = "tna"} variable "from_hw" {default = "False"} variable "setup_nodes_pb" {default = "setup_nodes.yml"} -variable "scenario_name" {default = "full"} +variable "scenario_name" {default = "lab_trial"} variable "test_case" {default = "all"} diff --git a/bin/bfrt_connect.py b/bin/bfrt_connect.py deleted file mode 100755 index 68f68651..00000000 --- a/bin/bfrt_connect.py +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2020 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import argparse -import logging -import sys - -import bfrt_grpc.client as bfrt_client - - -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument('-g', '--grpc-addr', required=False, - default='localhost:50052', - help='The GRPC address in the form host:port') - parser.add_argument('-n', '--program-name', required=False, - default=None, - help='The P4 program name') - parser.add_argument('-c', '--client-id', required=False, default=0, - type=int, help='The client ID - default 0') - parser.add_argument('-d', '--device-id', required=False, default=0, - type=int, help='The device ID - default 0') - parser.add_argument('-m', '--is-master', required=False, type=bool, - default=True, help='Is master client') - parser.add_argument('-t', '--table-name', required=False, - help='The table name for logging the ID') - parser.add_argument('-l', '--log-dir', type=str, required=False, - default=None) - parser.add_argument('-lf', '--log-file', type=str, required=False, - default='insert_p4_table.log') - parser.add_argument('-R', '--reset', '--clear', type=bool, required=False, - default=False, - help='Clear all tables before programming') - return parser.parse_args() - - -if __name__ == '__main__': - """ - Inserts entries into P4 tables based on the entries within table config - file - """ - args = get_args() - logger = logging.getLogger('bfrt_connect') - - # - # Set up Logging - # - if args.log_dir and args.log_file: - log_file = '{}/{}'.format(args.log_dir, args.log_file) - logging.basicConfig(level=logging.DEBUG, filename=log_file) - else: - logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) - - # - # Connect to the BF Runtime Server - # - logger.info('Creating client interface with client stub') - logger.debug('grpc-addr - [%s], client_id - [%s], device_id - [%s], ' - 'is_master - [%s]', - args.grpc_addr, args.client_id, args.device_id, - args.is_master) - - interface = bfrt_client.ClientInterface( - grpc_addr=args.grpc_addr, client_id=args.client_id, - device_id=args.device_id, is_master=args.is_master) - - # - # Get the information about the running program - # - bfrt_info = interface.bfrt_info_get(args.program_name) - - # Attempt to receive the ID for the args.table_name argument - if args.table_name: - logger.info('Retrieve table name - [%s]', args.table_name) - try: - table = bfrt_info.table_get(args.table_name) - logger.info('Table ID - [%s]', table.info.id_get()) - except Exception as e: - logger.error("Exit with error - [%s]", e) - - # - # A small workaround to close the connection properly. That should be - # addressed in the future versions of SDE - interface._tear_down_stream() - - # - # The End - # - logger.info('Exit 0') diff --git a/bin/bfrt_switch_tester.py b/bin/bfrt_switch_tester.py deleted file mode 100644 index 511f2b4b..00000000 --- a/bin/bfrt_switch_tester.py +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2020 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import argparse -import json -import logging -import sys - -import yaml - -from trans_sec.bfruntime_lib.aggregate_switch import AggregateSwitch -from trans_sec.bfruntime_lib.core_switch import CoreSwitch -from trans_sec.bfruntime_lib.gateway_switch import GatewaySwitch - - -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument('-t', '--topo', required=True, - help='The location of the topology file') - parser.add_argument('-n', '--program-name', required=False, - default=None, - help='The P4 program name') - parser.add_argument('-p', '--proto-dump-file', required=False, - default=None, help='The GRPC logger') - parser.add_argument('-c', '--client-id', required=False, default=0, - type=int, help='The client ID - default 0') - parser.add_argument('-l', '--log-dir', type=str, required=False, - default=None) - parser.add_argument('-lf', '--log-file', type=str, required=False, - default='insert_p4_table.log') - return parser.parse_args() - - -if __name__ == '__main__': - """ - Inserts entries into P4 tables based on the entries within table config - file - """ - args = get_args() - # - # Set up Logging - # - if args.log_dir and args.log_file: - log_file = '{}/{}'.format(args.log_dir, args.log_file) - logging.basicConfig(level=logging.DEBUG, filename=log_file) - else: - logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) - - logger = logging.getLogger('bfrt_switch_tester') - - topo_file = args.topo - with open(topo_file, 'r') as f: - if topo_file.endswith('json'): - topo = json.load(f) - else: - topo = yaml.safe_load(f) - - logger.info('topo - [%s]', topo) - - for sw_info in topo['switches'].values(): - logger.info('sw_info - [%s]', sw_info) - sw_type = sw_info['type'] - switch = None - if sw_type == 'core': - logger.info('Instantiating CoreSwitch') - switch = CoreSwitch(sw_info) - elif sw_type == 'aggregate': - logger.info('Instantiating AggregateSwitch') - switch = AggregateSwitch(sw_info) - elif sw_type == 'gateway': - logger.info('Instantiating GatewaySwitch') - switch = GatewaySwitch(sw_info, args.proto_dump_file) - - logger.info('Adding switch ID to switch - [%s]', sw_info['id']) - - if switch: - switch.add_switch_id(sw_info['id']) - - logger.info('Starting digest listeners') - switch.start_digest_listeners() - - logger.info('Stopping switch') - switch.stop_digest_listeners() - - # - # The End - # - logger.info('Exit 0') diff --git a/bin/bfrt_test.py b/bin/bfrt_test.py deleted file mode 100644 index db2eb144..00000000 --- a/bin/bfrt_test.py +++ /dev/null @@ -1,97 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2020 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import print_function - -import logging -import os -import sys - -import argparse -import bfrt_grpc.client as bfrt_client - -SDE_INSTALL = os.environ['SDE_INSTALL'] -SDE_PYTHON_27 = os.path.join(SDE_INSTALL, 'lib', 'python2.7', 'site-packages') - -sys.path.append(SDE_PYTHON_27) -sys.path.append(os.path.join(SDE_PYTHON_27, 'tofino')) - - -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument('-g', '--grpc-addr', required=True, - help='The GRPC address in the form host:port') - parser.add_argument('-i', '--ingress', required=True, type=str, - help='The name of the table to manipulate') - parser.add_argument('-d', '--device_id', required=False, type=int, - default=0, help='The name of the table to manipulate') - parser.add_argument('-t', '--table', required=False, type=str, - default='data_forward_t', - help='The name of the table to manipulate') - parser.add_argument('-a', '--action', required=False, type=str, - default='data_forward', - help='The name of the table to manipulate') - parser.add_argument('-f', '--mac-field', required=False, type=str, - default='hdr.ethernet.dst_mac', - help='The name of the table to manipulate') - parser.add_argument('-m', '--mac', required=False, type=str, - default='00:00:00:01:01:01', - help='The key value to the data_forward_t table') - parser.add_argument('-p', '--port', required=False, type=int, - default=2, - help='The acton value to the data_forward action') - return parser.parse_args() - - -if __name__ == '__main__': - """ - Inserts entries into P4 tables based on the entries within table config - file - """ - args = get_args() - logger = logging.getLogger('bfrt_connect') - logging.basicConfig(level=logging.DEBUG) - # - # Connect to the BF Runtime Server - # - interface = bfrt_client.ClientInterface( - grpc_addr=args.grpc_addr, - client_id=0, - device_id=0, - is_master=True) - logger.info('Connected to BF Runtime Server') - - target = bfrt_client.Target(device_id=args.device_id, pipe_id=0xffff) - # - # Get the information about the running program - # - bfrt_info = interface.bfrt_info_get() - p4_name = bfrt_info.p4_name_get() - logger.info('The target runs program ', p4_name) - interface.bind_pipeline_config(p4_name) - - table_name = "{}.{}".format(args.ingress, args.table) - logger.info('Table name - [%s]', table_name) - table = bfrt_info.table_get(table_name) - logger.info('Table class - [%s]', table.__class__) - table.info.key_field_annotation_add(args.mac_field, 'mac') - - action_name = "{}.{}".format(args.ingress, args.action) - - table.entry_add(target, - [bfrt_client.KeyTuple(args.mac_field, args.mac)], - [bfrt_client.DataTuple('port', val=args.port)]) - - interface._tear_down_stream() diff --git a/bin/call_sdn_api.py b/bin/call_sdn_api.py deleted file mode 100755 index f34d25fb..00000000 --- a/bin/call_sdn_api.py +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import argparse -import json -import logging - -import pydevd -import requests - -logger = logging.getLogger('oinc') -FORMAT = '%(levelname)s %(asctime)-15s %(filename)s %(lineno)d %(message)s' - - -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument( - '-l', '--loglevel', - help='Log Level defaults to INFO', - required=False, default='INFO') - parser.add_argument( - '-s', '--sdn-url', dest='sdn_url', required=False, - default='localhost:9998', help='the URL to the SDN controller') - parser.add_argument( - '-o', '--operation', dest='operation', required=False, - help='the operation to call', default='attack') - parser.add_argument( - '-p', '--protocol', dest='protocol', required=False, - help='the protocol to call', default='http') - - parser.add_argument( - '-dh', '--debug-host', dest='debug_host', - help='remote debugging host IP') - parser.add_argument( - '-dp', '--debug-port', dest='debug_port', default=5678, - help='the remote debugging port') - - attack = { - 'src_mac': '00:00:00:00:03:01', - 'src_ip': '10.0.1.6', - 'dst_ip': '10.0.1.8', - 'dst_port': '4323', - 'packet_size': '86', - 'attack_type': 'UDP Flood', - } - parser.add_argument('-a', '--rest-args', dest='rest_args', required=False, - default=attack, type=json.loads, - help='the REST call dict arguments') - return parser.parse_args() - - -def main(): - args = get_args() - - # Initialize logger - numeric_level = getattr(logging, args.loglevel.upper(), None) - logging.basicConfig(format=FORMAT, level=numeric_level) - - # Setup remote debugging - if args.debug_host: - pydevd.settrace(host=args.debug_host, port=int(args.debug_port), - stdoutToServer=True, stderrToServer=True, - suspend=False) - logger.info('Starting Oinc with SDN Controller url [%s]', args.sdn_url) - - logger.info('Retrieving http session from url - [%s]', args.sdn_url) - url = '{}://{}/{}'.format(args.protocol, args.sdn_url, args.operation) - ret_val = requests.post(url=url, params=args.rest_args) - logger.info('Return value of REST call [%s]', ret_val) - - -if __name__ == '__main__': - main() diff --git a/bin/full-gen-pcap-frg.sh b/bin/full-gen-pcap-frg.sh deleted file mode 100644 index 28bc31c0..00000000 --- a/bin/full-gen-pcap-frg.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash - -# Contains the packet generation and sniffing commands for generating INT and non-INT PCAP files - -# With INT -sudo python /home/ubuntu/transparent-security/trans_sec/device_software/send_packets.py -y 0 -i 0.005 -it 1 -itd 0 -z gateway1-eth1 -sa 192.168.1.2 -r 10.0.1.8 -sp 9648 -p 3074 -c 1 -e 00:00:00:00:01:01 -s 00:00:00:01:01:00 -pr UDP -m 'UDP INT IPv4' -sudo python /home/ubuntu/transparent-security/trans_sec/device_software/send_packets.py -y 0 -i 0.005 -it 1 -itd 0 -z gateway1-eth1 -sa 192.168.1.2 -r 10.0.1.8 -sp 9648 -p 3074 -c 1 -e 00:00:00:00:01:01 -s 00:00:00:01:01:00 -pr TCP -m 'TCP INT IPv4' - -sudo python /home/ubuntu/transparent-security/trans_sec/device_software/send_packets.py -y 0 -i 0.005 -it 1 -itd 0 -z gateway1-eth1 -sa 0000:0000:0000:0000:0000:0001:0001:0002 -r 0000:0000:0000:0000:0000:0002:0001:0002 -sp 9648 -p 3074 -c 1 -e 00:00:00:00:01:01 -s 00:00:00:01:01:00 -pr UDP -m 'UDP INT IPv6' -sudo python /home/ubuntu/transparent-security/trans_sec/device_software/send_packets.py -y 0 -i 0.005 -it 1 -itd 0 -z gateway1-eth1 -sa 0000:0000:0000:0000:0000:0001:0001:0002 -r 0000:0000:0000:0000:0000:0002:0001:0002 -sp 9648 -p 3074 -c 1 -e 00:00:00:00:01:01 -s 00:00:00:01:01:00 -pr TCP -m 'TCP INT IPv6' - -# Without INT -sudo python /home/ubuntu/transparent-security/trans_sec/device_software/send_packets.py -y 0 -i 0.005 -it 1 -itd 0 -z gateway1-eth1 -sa 192.168.1.2 -r 10.0.1.8 -sp 9648 -p 3074 -c 1 -e 00:00:00:00:05:05 -s 00:00:00:01:01:00 -pr UDP -m 'UDP Normal IPv4' -sudo python /home/ubuntu/transparent-security/trans_sec/device_software/send_packets.py -y 0 -i 0.005 -it 1 -itd 0 -z gateway1-eth1 -sa 192.168.1.2 -r 10.0.1.8 -sp 9648 -p 3074 -c 1 -e 00:00:00:00:05:05 -s 00:00:00:01:01:00 -pr TCP -m 'TCP Normal IPv4' - -sudo python /home/ubuntu/transparent-security/trans_sec/device_software/send_packets.py -y 0 -i 0.005 -it 1 -itd 0 -z gateway1-eth1 -sa 0000:0000:0000:0000:0000:0001:0001:0002 -r 0000:0000:0000:0000:0000:0002:0001:0002 -sp 9648 -p 3074 -c 1 -e 00:00:00:00:05:05 -s 00:00:00:01:01:00 -pr UDP -m 'UDP Normal IPv6' -sudo python /home/ubuntu/transparent-security/trans_sec/device_software/send_packets.py -y 0 -i 0.005 -it 1 -itd 0 -z gateway1-eth1 -sa 0000:0000:0000:0000:0000:0001:0001:0002 -r 0000:0000:0000:0000:0000:0002:0001:0002 -sp 9648 -p 3074 -c 1 -e 00:00:00:00:05:05 -s 00:00:00:01:01:00 -pr TCP -m 'TCP Normal IPv6' - - -# Commands for sniffing the important interfaces while generating the packets above -sudo tcpdump -i gateway1-eth1 -w gateway1-eth1.pcap -sudo tcpdump -i gateway1-eth2 -w gateway1-eth2.pcap -sudo tcpdump -i aggregate-eth1 -w aggregate-eth1.pcap -sudo tcpdump -i aggregate-eth2 -w aggregate-eth2.pcap -sudo tcpdump -i core-eth2 -w core-eth2.pcap -sudo tcpdump -i core-eth3 -w core-eth3.pcap diff --git a/bin/run_p4_mininet.py b/bin/run_p4_mininet.py deleted file mode 100755 index e452a002..00000000 --- a/bin/run_p4_mininet.py +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import argparse -import json -import logging -import sys -import yaml - -from trans_sec.mininet.exercise import ExerciseRunner - -logger = logging.getLogger('') - - -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument('-t', '--topo', help='Path to topology json', - type=str, required=True) - parser.add_argument('-l', '--log-dir', type=str, required=True, - default=None) - parser.add_argument('-lf', '--log-file', type=str, required=False, - default='run_p4_mininet.log') - parser.add_argument('-p', '--pcap-dir', type=str, required=False, - default=None) - parser.add_argument('-j', '--switch_json', type=str, required=False) - parser.add_argument('-c', '--start-cli', type=bool, required=False, - default=None) - parser.add_argument('-d', '--daemon', help='Run device daemon on hosts.', - type=bool, required=False, default=False) - return parser.parse_args() - - -def read_yaml_file(config_file_path): - """ - Reads a yaml file and returns a dict representation of it - :return: a dict of the yaml file - """ - logger.debug('Attempting to load configuration file - ' + config_file_path) - config_file = None - try: - with open(config_file_path, 'r') as config_file: - config = yaml.safe_load(config_file) - logger.info('Loaded configuration') - return config - finally: - if config_file: - logger.info('Closing configuration file') - config_file.close() - - -if __name__ == '__main__': - args = get_args() - - if args.log_file: - log_file = '{}/{}'.format(args.log_dir, args.log_file) - logging.basicConfig(level=logging.DEBUG, filename=log_file) - else: - logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) - - topo_file = args.topo - if topo_file.endswith('json'): - with open(topo_file, 'r') as f: - topo = json.load(f) - else: - topo = read_yaml_file(topo_file) - - exercise = ExerciseRunner( - topo, args.log_dir, args.pcap_dir, args.switch_json, args.start_cli) - exercise.run_exercise() - - logger.info('Exercise Runner running indefinitely') - while True: - pass diff --git a/bin/sdn_controller.py b/bin/sdn_controller.py index 8a3171bd..027976a9 100755 --- a/bin/sdn_controller.py +++ b/bin/sdn_controller.py @@ -22,8 +22,7 @@ from trans_sec.controller.aggregate_controller import AggregateController from trans_sec.controller.core_controller import CoreController from trans_sec.controller.ddos_sdn_controller import ( - DdosSdnController, GATEWAY_CTRL_KEY, AGG_CTRL_KEY, CORE_CTRL_KEY) -from trans_sec.controller.gateway_controller import GatewayController + DdosSdnController, AGG_CTRL_KEY, CORE_CTRL_KEY) FORMAT = '%(levelname)s %(asctime)-15s %(filename)s %(message)s' logger = logging.getLogger('sdn_controller') @@ -50,8 +49,7 @@ def get_args(): parser.add_argument('-ld', '--log-dir', type=str, required=True, default=default_logs) parser.add_argument('-t', '--topo', help='Path to topology json', - required=False, - default='../mininet-start/conf/topology_proposed.json') + required=True) parser.add_argument('-p', '--platform', help='Switch Platform', required=True, type=str, choices=['bmv2', 'tofino']) parser.add_argument('-s', '--switch-config-dir', dest='switch_config_dir', @@ -78,8 +76,7 @@ def get_args(): help='When set, the controller will not attempt to ' 'load the P4 program onto the switches') parser.add_argument('-ct', '--controller-type', type=str, required=True, - choices=['gateway', 'aggregate', 'core', 'full', - 'lab_trial'], + choices=['aggregate', 'core', 'full', 'lab_trial'], help='When not set, all controllers will attempt to ' 'start') return parser.parse_args() @@ -122,15 +119,6 @@ def main(): 'Starting SDN Controller with topology - [%s] and load_p4 flag - [%s]', topo, eval(args.load_p4)) controllers = dict() - if (args.controller_type == 'gateway' - or args.controller_type == 'full'): - logger.info('Instantiating a GatewayController') - controllers[GATEWAY_CTRL_KEY] = GatewayController( - platform=args.platform, - p4_build_out=args.switch_config_dir, - topo=topo, - log_dir=args.log_dir, - load_p4=eval(args.load_p4)) if (args.controller_type == 'aggregate' or args.controller_type == 'lab_trial' or args.controller_type == 'full'): diff --git a/bin/send.py b/bin/send.py deleted file mode 100755 index 1c55972c..00000000 --- a/bin/send.py +++ /dev/null @@ -1,130 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import argparse -import logging -import random -import socket -from logging import getLogger, basicConfig -from time import sleep - -from scapy.all import get_if_list, get_if_hwaddr, get_if_addr -from scapy.layers.inet import IP, UDP, TCP -from scapy.layers.l2 import Ether -from scapy.sendrecv import sendp - -# Logger stuff -logger = getLogger('send') - -FORMAT = '%(levelname)s %(asctime)-15s %(filename)s %(message)s' - - -def get_if(target): - iface = None - logger.error(target) - for i in get_if_list(): - logger.info(i.find(target)) - if i.find(target) >= 0: - iface = i - break - if not iface: - logger.error('Cannot find %s interface' % target) - exit(1) - return iface - - -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument('-d', '--duration', help='Number of seconds to run', - type=int, required=True) - parser.add_argument('-i', '--interval', - help='How often to send packets in seconds', - type=float, required=True) - parser.add_argument('-y', '--delay', help='Delay before starting run', - type=int, required=False, default=0) - parser.add_argument('-r', '--destination', help='Destination IPv4 address', - required=True) - parser.add_argument('-sa', '--source-addr', help='Source address') - parser.add_argument('-sp', '--source-port', type=int, - help='Source port else it will be random') - parser.add_argument('-p', '--port', help='Destination port', type=int, - required=True) - parser.add_argument('-m', '--msg', help='Message to send', required=True) - parser.add_argument('-l', '--loglevel', - help='Log Level defaults ' - 'to INFO', - required=False, default='INFO') - parser.add_argument('-f', '--logfile', - help='File to log to defaults to console', - required=False, default=None) - parser.add_argument('-z', '--interface', - help='Linux named ethernet device. Defaults to eth0', - required=False, default='eth0') - parser.add_argument('-s', '--switch_ethernet', - help='Switch Ethernet Interface. Defaults to ' - 'ff:ff:ff:ff:ff:ff', - required=False, default='ff:ff:ff:ff:ff:ff') - parser.add_argument('-t', '--tcp', dest='tcp', action='store_true', - required=False) - args = parser.parse_args() - return args - - -def device_send(args): - numeric_level = getattr(logging, args.loglevel.upper(), None) - basicConfig(format=FORMAT, level=numeric_level, filename=args.logfile) - addr = socket.gethostbyname(args.destination) - count = int((args.duration - args.delay) / args.interval) - interface = get_if(args.interface) - logger.info('Delaying %d seconds' % args.delay) - sleep(args.delay) - - logger.info( - 'sending %s packets at %s sec/packet on interface %r to %s for %s ' - 'seconds', - count, args.interval, interface, args.destination, - (args.duration - args.delay)) - logger.info('SRC ADDR for interface %s - %s', interface, - get_if_hwaddr(interface)) - pkt = Ether(src=get_if_hwaddr(interface), dst=args.switch_ethernet) - logger.info('packet Ether obj - %s', pkt) - - if args.source_addr: - src_ip = args.source_addr - else: - src_ip = get_if_addr(interface) - - if args.source_port: - src_port = args.source_port - else: - src_port = random.randint(49152, 65535) - - if args.tcp: - pkt = pkt / IP(dst=addr, src=src_ip) / TCP(dport=args.port, - sport=src_port) / args.msg - else: - pkt = pkt / IP(dst=addr, src=src_ip) / UDP(dport=args.port, - sport=src_port) / args.msg - - pkt.show2() - sendp(pkt, iface=interface, verbose=False, count=count, - inter=args.interval) - logger.info('Done') - return - - -if __name__ == '__main__': - logger.info('Starting Send') - cmd_args = get_args() - device_send(cmd_args) diff --git a/bin/start_ae.py b/bin/start_ae.py deleted file mode 100755 index 8d50109f..00000000 --- a/bin/start_ae.py +++ /dev/null @@ -1,117 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import argparse -import logging -import sys - -import pydevd - -from trans_sec.analytics.oinc import Oinc, SimpleAE, LoggerAE, IntLoggerAE -from trans_sec.utils.http_session import HttpSession - -logger = logging.getLogger('start_ae') -FORMAT = '%(levelname)s %(asctime)-15s %(filename)s %(lineno)d %(message)s' - - -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument( - '-l', '--loglevel', - help='Log Level defaults to INFO', - required=False, default='INFO') - parser.add_argument('-f', '--logfile', - help='File to log to defaults to console', - required=False, default=None) - parser.add_argument('-i', '--interface', - help='Linux interface to listen on', required=True) - parser.add_argument('-di', '--drop-interface', dest='drop_interface', - help='Linux interface to listen for Drop Reports', - required=False) - parser.add_argument('-dh', '--debug-host', dest='debug_host', - help='remote debugging host IP') - parser.add_argument('-dp', '--debug-port', dest='debug_port', default=5678, - help='the remote debugging port') - parser.add_argument('-s', '--sdn-url', dest='sdn_url', required=True, - help='the URL to the SDN controller') - parser.add_argument('-t', '--type', dest='type', required=True, - choices=['OINC', 'SIMPLE', 'LOGGING', 'INT'], - help='Acceptable values OINC|SIMPLE|LOGGING|INT') - parser.add_argument('-c', '--sdn-attack-ctx', dest='sdn_ctx', - required=False, default='gwAttack', - help='the URL to the SDN controller') - parser.add_argument('-pc', '--packet-count', required=False, type=int, - default=100, - help='Number of packets to hit alert within the ' - 'sample-interval') - parser.add_argument('-si', '--sample-interval', required=False, type=int, - default=60, - help='Number of seconds the packet count needs to hit ' - 'to trigger alert') - parser.add_argument('-pb', '--packet-bytes', required=False, type=int, - default=50000, - help='Number of packet bytes to hit alert within the ' - 'sample-interval') - return parser.parse_args() - - -def main(): - args = get_args() - - # Initialize logger - numeric_level = getattr(logging, args.loglevel.upper(), None) - - if args.logfile: - logging.basicConfig(format=FORMAT, level=numeric_level, - filename=args.logfile) - else: - logging.basicConfig(format=FORMAT, level=numeric_level) - - # Setup remote debugging - if args.debug_host: - pydevd.settrace(host=args.debug_host, port=int(args.debug_port), - stdoutToServer=True, stderrToServer=True, - suspend=False) - logger.info('Starting Oinc with SDN Controller url [%s]', args.sdn_url) - - ae = None - - logger.info('Retrieving http session from url - [%s]', args.sdn_url) - http_session = HttpSession(args.sdn_url) - - if args.type == 'SIMPLE': - logger.info('SimpleAE instantiated') - logger.debug('SDN Context - [%s]', args.sdn_ctx) - ae = SimpleAE(http_session, packet_count=args.packet_count, - sample_interval=args.sample_interval, - sdn_attack_context=args.sdn_ctx, - byte_count=args.packet_bytes) - elif args.type == 'OINC': - logger.info('Oinc instantiated') - ae = Oinc(http_session) - elif args.type == 'LOGGING': - logger.info('LoggerAE instantiated') - ae = LoggerAE(http_session) - elif args.type == 'INT': - logger.info('LoggerAE instantiated') - ae = IntLoggerAE(http_session) - - logger.info('Begin sniffing for INT on [%s] and Drop Reports on [%s]', - args.interface, args.drop_interface) - ae.start_sniffing(args.interface, args.drop_interface) - sys.stdout.flush() - - -if __name__ == '__main__': - main() diff --git a/bin/write_to_clone.py b/bin/write_to_clone.py deleted file mode 100755 index 175aac2d..00000000 --- a/bin/write_to_clone.py +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import argparse -import logging -import sys - -from trans_sec.p4runtime_lib.p4rt_switch import P4RuntimeSwitch -from trans_sec.p4runtime_lib.helper import P4InfoHelper -from trans_sec.switch import SwitchConnection - -logger = logging.getLogger('insert_p4_clone_session_entry') - - -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument('-a', '--grpc_addr', type=str, required=True, - help='Switch grpc host:port') - parser.add_argument('-d', '--dev_id', type=int, required=False, default=0, - help='The device ID (default 0)') - parser.add_argument( - '-p', '--p4-info-fpath', type=str, required=False, default=0, - help='The file path of the switch associated p4info file') - parser.add_argument('-l', '--log-dir', type=str, required=False, - default=None) - parser.add_argument('-lf', '--log-file', type=str, required=False, - default='insert_p4_table.log') - parser.add_argument('-c', '--clone', type=str, required=False, - default='True', help='Build clone entry') - parser.add_argument('-ce', '--clone-egress', type=int, required=False, - default=0, help='Clone egress port') - return parser.parse_args() - - -if __name__ == '__main__': - """ - Inserts clone entries based on the table config file - """ - args = get_args() - if args.log_dir and args.log_file: - log_file = '{}/{}'.format(args.log_dir, args.log_file) - logging.basicConfig(stream=sys.stdout, level=logging.DEBUG, - filename=log_file) - else: - logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) - - logger.info('Connecting to BMV2 Switch #[%s] at [%s] loaded with P4 [%s]', - args.dev_id, args.grpc_addr, args.p4_info_fpath) - p4info_helper = P4InfoHelper(args.p4_info_fpath) - - # TODO/FIXME - This will break and need to know which type of switch - # connection to start - switch = SwitchConnection( - name='test', address=args.grpc_addr, device_id=args.dev_id) - switch.master_arbitration_update() - - if args.clone == 'True': - clone_entry = p4info_helper.build_clone_entry(args.clone_egress) - logger.info('Inserting clone entry [%s]', clone_entry) - switch.write_clone_entries(clone_entry) - else: - clone_entry = p4info_helper.build_clone_entry(args.clone_egress) - logger.info('Deleting clone entry [%s]', clone_entry) - switch.delete_clone_entries(clone_entry) diff --git a/bin/write_to_p4_table.py b/bin/write_to_p4_table.py deleted file mode 100755 index 4df302ff..00000000 --- a/bin/write_to_p4_table.py +++ /dev/null @@ -1,146 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import argparse -import logging -import sys - -import yaml - -from trans_sec.p4runtime_lib.gateway_switch import GatewaySwitch -from trans_sec.p4runtime_lib.aggregate_switch import AggregateSwitch -from trans_sec.p4runtime_lib.core_switch import CoreSwitch -from trans_sec.p4runtime_lib.helper import P4InfoHelper - -logger = logging.getLogger('insert_p4_table_entry') - - -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument('-s', '--switch-name', required=True, - help='The switch name to control') - parser.add_argument('-t', '--topo', required=True, - help='The location to the topology file') - parser.add_argument('-n', '--ingress', required=True, - help='The P4 ingress classname') - parser.add_argument( - '-i', '--insert', type=str, required=False, default='True', - help='When true insert record else delete (default "True")') - parser.add_argument('-tc', '--table-config', - help='Table insertion config file', - type=str, required=False) - parser.add_argument('-e', '--election-id', - help='Table insertion config file', - type=int, required=False, default=0) - parser.add_argument('-l', '--log-dir', type=str, required=False, - default=None) - parser.add_argument('-lf', '--log-file', type=str, required=False, - default='insert_p4_table.log') - return parser.parse_args() - - -def read_yaml_file(config_file_path): - """ - Reads a yaml file and returns a dict representation of it - :return: a dict of the yaml file - """ - logger.debug('Attempting to load configuration file - ' + config_file_path) - config_file = None - try: - with open(config_file_path, 'r') as config_file: - config = yaml.safe_load(config_file) - logger.info('Loaded configuration') - return config - finally: - if config_file: - logger.info('Closing configuration file') - config_file.close() - - -if __name__ == '__main__': - """ - Inserts entries into P4 tables based on the entries within table config - file - """ - args = get_args() - if args.log_dir and args.log_file: - log_file = '{}/{}'.format(args.log_dir, args.log_file) - logging.basicConfig(level=logging.DEBUG, filename=log_file) - else: - logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) - - table_entry_config = read_yaml_file(args.table_config) - topo = read_yaml_file(args.topo) - sw_info = topo['switches'][args.switch_name] - - logger.info('Connecting to BMV2 Switch #[%s]', sw_info) - p4info_helper = P4InfoHelper(sw_info['runtime_p4info']) - - for entry_config in table_entry_config: - logger.info('Entry config - [%s]', entry_config) - logger.info('Writing table entry - [%s] for insert - [%s]', - entry_config, args.insert) - - switch = None - if sw_info['type'] == 'gateway': - switch = GatewaySwitch(sw_info) - elif sw_info['type'] == 'aggregate': - switch = AggregateSwitch(sw_info) - elif sw_info['type'] == 'core': - switch = CoreSwitch(sw_info) - - if not switch: - raise Exception('Switch type of [%s] is not supported', - sw_info['type']) - switch.master_arbitration_update() - - match_fields = dict() - if 'match_fields' in entry_config: - logger.info('Match fields obj - [%s]', - entry_config['match_fields']) - for key, value in entry_config['match_fields'].items(): - if isinstance(value, list) or isinstance(value, tuple): - logger.debug('Match fields list - [%s]', value) - match_fields[key] = (value[0], value[1]) - else: - logger.debug('Match fields list - [%s]', value) - match_fields[key] = value - - if args.insert == 'True': - entries = switch.get_match_values(entry_config['table_name']) - logger.info('Existing table entries - {}'.format(entries)) - logger.info( - 'Entry configuration for table entry - [%s] and match fields ' - '- [%s]', entry_config, match_fields) - table_entry = p4info_helper.build_table_entry( - table_name=entry_config['table_name'], - match_fields=match_fields, - action_name=entry_config.get('action_name'), - action_params=entry_config.get('action_params'), - ) - logger.debug( - 'Writing table entry to table [%s], with action name - [%s], ' - 'match fields - [%s], action_params - [%s]', - entry_config['table_name'], entry_config.get('action_name'), - match_fields, entry_config.get('action_params')) - switch.write_table_entry(table_entry, - election_high=args.election_id, - election_low=args.election_id + 1) - else: - table_entry = p4info_helper.build_table_entry( - table_name=entry_config['table_name'], - match_fields=entry_config.get('match_fields'), - ) - logger.info('Deleting table entry') - switch.delete_table_entry(table_entry) diff --git a/docs/BUILD.md b/docs/BUILD.md deleted file mode 100644 index 269b5008..00000000 --- a/docs/BUILD.md +++ /dev/null @@ -1,465 +0,0 @@ -# transparent-security automation - -The scripts outlined here have been designed to be executed within a CI server - -## Table of Contents - -1. Introduction -2. Client system setup -3. [Optional] Create an OS instance for running the mininet simulator -4. Run mininet simulator (using Terraform) -5. [Optional] Run mininet simulator (using ansible) -6. [Optional] Run mininet simulator on a local VM -7. Using Mininet - -## 1. Introduction - -This document provides instructuions to: - -1. Building a transparent-security environment on AWS -1. Setting up Mininet and AWS - -## 2. Client system setup - -When running builds and simulations this project recommends running them on a cloud infrastructure. These instructions are using AWS EC2. With minor changes these could be run on other cloud types. - -You will use a local system for: - -* Running Ansible and Terraform to orchestrate the simulator -* Downloading the Transparent Security source -* Configuring the input file - -The local system can be Linux, Mac OS or Windows. We provide examples for Linux. This has also been testing on Mac OS. - -### 2.1 Install dependencies on local client - -Install git, python-ansible and terraform. - -#### 2.1.1 Install git - -Install the git client. - -On Ubuntu: - -```bash -sudo apt update -sudo apt install git -``` - -#### 2.1.2 Install Python Ansible - -* Python 3.6+ is installed -* The python3-pip package has been installed -* The Python ansible >=2.7.5 package has been installed - -Use python3-pip to install ansible >=2.7.5. - -On Ubuntu run: - -```bash -sudo apt update -sudo apt install python3-pip -sudo pip install ansible -``` - -Validate the ansible version: - -```bash -ansible --version -``` - -Example output: - -```bash -ansible 2.9.2 -. -. -. - python version = 3.6.8 (default, Nov 9 2019, 05:55:08) [GCC 4.2.1 Compatible Apple LLVM 11.0.0 (clang-1100.0.32.4) (-macos10.15-objc-s -``` - -#### 2.1.2 Install Terraform - -See terraform documentation for installation instructions. - -[Terraform Download page](https://www.terraform.io/downloads.html) - -### 2.2. Download Transparent Secuirty from Git - -Download the latest source from [Transparent Security GitHub](https://github.com/cablelabs/transparent-security) - -```bash -git clone https://github.com/cablelabs/transparent-security -``` - -Example output: - -```bash -Cloning into 'transparent-security'... -. -. -. -Resolving deltas: 100% (554/554), done. -``` - -### 2.3. Obtain credentials to AWS - -To use the directions, you will need an account on [AWS](https://aws.amazon.com/) with API access keys. - -### 2.4 Customize the variable file for your environment - -Copy the example variable file docs/mininet-example.tfvars to a working directory and make changes to adapt the file to your local environment. - -| Variable | Description | Type | Example | -|------------------|-------------------------------------------------------------------------------------------------------------------------------------------|--------|---------------------------------------------------------| -| build_id | This value must be unique to ensure multiple jobs can be run simultaneously from multiple hosts | string | build_id = "test-1" | -| access_key | Amazon EC2 access key | string | access_key = "AKIAIOSFODNN7EXAMPLE" | -| secret_key | Amazon EC2 secret key | string | secret_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" | -| ec2_region | Amazon EC2 region | string | ec2_region = "us-west-2" | -| public_key_file | Used to inject into the VM for SSH access with the user'ubuntu' (defaults to ~/.ssh/id_rsa.pub) | string | public_key_file = "~/.ssh/id_rsa.pub" | -| private_key_file | Used to access the VM via SSH with the user 'ubuntu' (defaults to ~/.ssh/id_rsa) | string | private_key_file = "~/.ssh/id_rsa" | -| env_type | The type of environemnt being built (only used for creating the environment) | string | env_type = "mininet" | -| mininet_ami | The AMI for the mininet environment (defaults to "ami-060d055b5ca40de8c"). Only used for running the simulator. | string | mininet_ami = "ami-060d055b5ca40de8c" | -| create_ami | When 'yes', the an EC2 image (AMI) will be created. | string | create_ami = "yes" | - -## 3. [Optional] Create an OS instance for running the mininet simulator - -This step is optional if you are running on AWS and use the AMI provided by CableLabs. - -Section 3.1 provides instructions for using Terraform to build an AMI on your AWS. - -Section 3.2 provides instructions for building an image in another environment or on baremetal. - -### 3.1. Build an AMI for running mininet on AWS - -#### 3.1.1 Create VM with Terraform - -This step will creat an VM on AWS, install all mininet dependencies and create an AMI. - -```bash -cd transparent-security/automation/env-build -terraform init -terraform apply -auto-approve -var-file="/path/to/my-mininet.tfvars" -``` - -Sample Output: - -```bash -aws_key_pair.transparent-security-mini-pk: Creating... -aws_security_group.transparent-security-img-sg: Creating... -aws_key_pair.transparent-security-mini-pk: Creation complete after 1s -. -. -. -Apply complete! Resources: 12 added, 0 changed, 0 destroyed. - -Outputs: - -ami-id = ami-0393652ac3fbc331e -ip = 34.211.114.181 -``` - -Save the ami-id and it to your variables file. - -#### 3.1.2 Remove the AMI for terraform state - -Remove the AMI from the terraform state so that it will remain after destroying the VM. - -```bash -terraform state rm aws_ami_from_instance.transparent-security-env-build -Removed aws_ami_from_instance.transparent-security-env-build -Successfully removed 1 resource instance(s). -``` - -#### 3.1.3 Clean up the VM used to create the AMI - -This step will remove everything except the AMI that was used to create the VM. - -```bash -terraform destroy -auto-approve -var-file="/path/to/my-mininet.tfvars" -``` - -Sample output: - -```bash -aws_key_pair.transparent-security-mini-pk: Refreshing state... [id=terraform-20191213203053435500000001] -aws_security_group.transparent-security-img-sg: Refreshing state... [id=sg-057e54e0162c6251a] -. -. -. -Destroy complete! Resources: 4 destroyed. -``` - -## 4. Run mininet simulator (using Terraform) - -Use the environment file created in section 2.4 - -### 4.1 Run terraform to launch the simulator on AWS. - -```bash -cd transparent-security/automation/p4/mininet -terraform init -terraform apply -auto-approve -var-file="/path/to/my-mininet.tfvars" -``` - -Sample Output: - -```bash -aws_key_pair.transparent-security-mini-pk: Creating... -aws_security_group.transparent-security-img-sg: Creating... -aws_key_pair.transparent-security-mini-pk: Creation complete after 5s -. -. -. -Apply complete! Resources: 12 added, 0 changed, 0 destroyed. - -Outputs: - -ip = 34.211.54.181 -``` - -### 4.2 Obtain Deployment Information - -```bash -# from transparent-security/automation/p4/mininet directory -terraform show -``` - -Sample output - - -```bash -# aws_instance.transparent-security-mininet-integration: -resource "aws_instance" "transparent-security-mininet-integration" { - ami = -. -. -. -. -Outputs: - -ip = "34.211.114.181" -``` - -### 4.3 SSH into EC2 Mininet VM - -Login to the VM running the simulator. Use the SSH keys indicated in the variable file to login -to the VM. - -```bash -# from transparent-security/automation/p4/mininet directory -ssh -i ubuntu@$(terraform output ip) -``` - -Sample output - - -```bash -Welcome to Ubuntu 18.04.5 LTS (GNU/Linux 4.4.0-1075-aws x86_64) - - * Documentation: https://help.ubuntu.com - * Management: https://landscape.canonical.com - * Support: https://ubuntu.com/advantage - - Get cloud support with Ubuntu Advantage Cloud Guest: - http://www.ubuntu.com/business/services/cloud - -149 packages can be updated. -89 updates are security updates. - -New release '18.04.3 LTS' available. -Run 'do-release-upgrade' to upgrade to it. - - -Last login: Wed Dec 11 22:39:13 2019 from 127.0.0.1 -ubuntu@ip-172-31-15-5:~$ -``` - -Upgrading to a newer version of Ubuntu isn't currently supported. Do so at your own risk. - -### 4.4. Development and debugging of Python - -The playbooks will be installing the python code located in the trans_sec -directory into the VM's Python runtime in place so any changes there will be -realized immediately. - -### 4.5. Cleanup the simulation environment - -This will remove the VM and other artifacts created when it was deployed. - -```bash -# from transparent-security/automation/p4/mininet directory -terraform destroy -auto-approve -var-file="/path/to/my-mininet.tfvars" -``` - -Sample output: - -```bash -. -. -Destroy complete! Resources: 12 destroyed. - -Process finished with exit code 0 -``` - -## 5. [Optional] Run mininet simulator (using ansible) - -### 5.1. Launch an AWS instance using pre-built AMI - -- From the EC2 dashboard, launch a new instance with the AMI provided by CableLabs. -- Configure the security group for the instance as follows- - -| Type | Protocol | Port Range | Source | -|:---------------:|:--------:|:----------:|:---------:| -| Custom TCP Rule | TCP | 8080 | 0.0.0.0/0 | -| SSH | TCP | 22 | 0.0.0.0/0 | -| Custom TCP Rule | TCP | 3000 | 0.0.0.0/0 | -| HTTPS | TCP | 443 | 0.0.0.0/0 | - -- Review and launch the instance. When prompted, generate a new key-pair and save it on the local machine. - -### 5.2. Create a local ansible inventory - -Generate your local ansible inventory file with the following command where - is the name of the output inventory file to be used -in step 5.4 below and is the IP of the host you want to setup. -```bash -ansible-playbook transparent-security/playbooks/mininet/local_inventory.yml \ ---extra-vars "public_ip= local_inventory=" -``` - -### 5.3. Create and inject own SSH keys - -- Login to the remote VM -```bash -ssh -i ubuntu@ -``` -- Create and inject SSH keys to be able to access the mininet hosts -```bash -ssh-keygen -t rsa -N '' -f ~/.ssh/id_rsa -touch ~/.ssh/authorized_keys -chmod 600 ~/.ssh/authorized_keys -cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys -``` - -### 5.4. Setup transparent-security directory and install dependencies on the remote VM - -- On the local machine, run the following command to setup the mininet host - -```bash -export ANSIBLE_HOST_KEY_CHECKING=False -ansible-playbook -u ubuntu -i transparent-security/playbooks/mininet/setup_host.yml --key-file ~/.ssh/id_rsa -``` - -### 5.5. Start mininet simulation - -- On the remote VM, install ansible before proceeding to begin simulation. -```bash -sudo pip install ansible -export ANSIBLE_HOST_KEY_CHECKING=False -ansible-playbook -u ubuntu -i transparent-security.ini transparent-security/playbooks/mininet/setup_mininet.yml -``` -Note - The transparent-security.ini refers to the inventory file on the remote machine which is generated in Step 5.4. - -### 5.6. Test with an attack scenario - -- On the remote VM, execute the attack scenario to validate attack detection and mitigation. -- To use the sample scenario provided by CableLabs, run the following command on the remote VM - -```bash -export ANSIBLE_HOST_KEY_CHECKING=False -ansible-playbook -u ubuntu -i transparent-security.ini transparent-security/playbooks/scenarios/full/all.yml -``` -Note - Refer the Wiki page [Attack Scenario](https://github.com/cablelabs/transparent-security/wiki/2.-Attack-Scenario) for a -detailed explanation of the attack scenario. - -## 6. [Optional] Run mininet simulator on a local VM - -For this purpose, we use a Ubuntu 18.04 VirtualBox VM running on the local machine. - -### 6.1. Setup the local VM - -- The network settings for the VM would have to allow access to the internet (to install -dependencies) and a channel for communication with the local machine. - - Bridged Adapter - - Host-only Adapter - -- Enable SSH on the VM and verify the SSH service to be active -```bash -sudo apt-get install openssh-server -sudo service ssh status -``` - -- Allow passwordless sudo by editing the /etc/sudoers file -```bash -sudo visudo -%sudo ALL=(ALL:ALL) NOPASSWD: ALL -``` -- Install Python packages -```bash -sudo apt-get update -sudo apt-get install python3.6 -y -sudo ln /usr/bin/python3.6 /usr/bin/python -``` -- Create and inject SSH keys to be able to access the mininet hosts -```bash -ssh-keygen -t rsa -N '' -f ~/.ssh/id_rsa -touch ~/.ssh/authorized_keys -chmod 600 ~/.ssh/authorized_keys -cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys -``` - -### 6.2. Setup the local machine - -- Copy the SSH key to the VM. On the local machine, -```bash -ssh-copy-id -i VM_user@VM_host -``` - -- Create a inventory file on the local machine to configure variables and VM IP address. -Note - Copy the example inventory file docs/example-local-inventory.ini to a working directory and - make changes to adapt the file to your local environment. - -### 6.3. Build the environment to run mininet simulation - -- On the local machine, run the env_build.yml to install the necessary software packages to run mininet. -```bash -export ANSIBLE_HOST_KEY_CHECKING=False; ansible-playbook -u ubuntu -i ~/variables.ini playbooks/mininet/env-build.yml -``` -Note - The env-build approximately takes 45-60 minutes to finish. - -### 6.4. Setup transparent-security directory and install dependencies on the VM - - - On the local machine, run the following command to create the inventory file - for setting up the mininet host: - ```bash -ansible-playbook transparent-security/playbooks/mininet/local_inventory.yml \ ---extra-vars "public_ip= local_inventory=" - ``` - - - On the local machine, run the following command to setup the mininet host: - ```bash - export ANSIBLE_HOST_KEY_CHECKING=False - ansible-playbook -u ubuntu -i transparent-security/playbooks/mininet/setup_host.yml --key-file ~/.ssh/id_rsa - ``` -### 6.5. Start mininet simulation - -- On the VM, install ansible before proceeding to begin simulation. -```bash -sudo pip install ansible -export ANSIBLE_HOST_KEY_CHECKING=False -ansible-playbook -u ubuntu -i transparent-security.ini transparent-security/playbooks/mininet/setup_mininet.yml -``` -Note - The transparent-security.ini refers to the inventory file on the remote machine which is generated in Step 6.4. - -### 6.6. Test with the UDP Flood attack scenario - -- On the VM, execute the attack scenario to validate attack detection and mitigation. -- To use the sample scenario provided by CableLabs, run the following command on the remote VM - -```bash -export ANSIBLE_HOST_KEY_CHECKING=False -ansible-playbook -u ubuntu -i transparent-security.ini transparent-security/playbooks/scenarios/full/all.yml -``` -Note - Refer the Wiki page [Attack Scenario](https://github.com/cablelabs/transparent-security/wiki/2.-Attack-Scenario) for a -detailed explanation of the attack scenario. - -## 7. Using Mininet - -- Refer the Wiki page [Using Mininet](https://github.com/cablelabs/transparent-security/wiki/3.-Using-Mininet) for -information on the default mininet architecture, how to access devices and capture packet-level data. diff --git a/docs/ae/SIDDHI_AE_SETUP.md b/docs/ae/SIDDHI_AE_SETUP.md new file mode 100644 index 00000000..0700da0a --- /dev/null +++ b/docs/ae/SIDDHI_AE_SETUP.md @@ -0,0 +1,97 @@ +# TPS Siddhi AE + +The Transparent Security Analytics Engine (TPS AE) is an analytics engine +implementation based on [Siddhi](https://siddhi.io/) event stream processing +(ESP) and complex event processing (CEP) framework. The Siddhi scripts +developed are able to identify a DDoS attack in less +than 1 second by examining telemetry reports generated by a switch or router. +This will also identify when a mitigated attack has ended based on drop +telemetry reports from the SDN controller. + +The directions below are how to run this as a simple Java program (as with our +"lab_trial" scenario tests), but these scripts can also be deployed to any +Siddhi compliant engine containing our +[UDP](https://github.com/cablelabs/siddhi-io-udp) & +[P4-TRPT](https://github.com/cablelabs/siddhi-map-p4-trpt) extensions +(and a running Kafka instance). Additionally, Siddhi scripts can also be +converted to run in Kubernetes where the siddhi-operator has been installed. + +See diagrams of the Siddhi scripts as generated by the WSO2 Streaming +Integrator Tooling: + +![image](udp-src-trpt.png) +UDP Packet Telemetry Report Source to Kafka JSON + +![image](kafka-src-pkt.png) +DDoS detect + +![image](kafka-src-drop.png) +DDoS drop + + +## Host Software Requirements + +1. Java 8+ +1. git +1. maven +1. kafka + +## Setup and run + +There are several methods that can be used to run Siddhi scripts. + +### Java application +The top-level Ansible playbook performing the actions listed below can be found +[here](../../playbooks/siddhi/setup_siddhi_maven.yml) + +- Install Java 8+ - [playbook](../../playbooks/siddhi/setup_jdk.yml) (setting up OpenJDK 11.0.11) +- Install and start Kafka to port 9092 - [playbook](../../playbooks/siddhi/setup_kafka.yml) + [installed based on these instructions](https://www.digitalocean.com/community/tutorials/how-to-install-apache-kafka-on-ubuntu-20-04) +- Install Siddhi for Maven - [playbook](../../playbooks/siddhi/setup_siddhi_p4.yml) + - Install Git client (apt install git) + - Install Maven client (apt install maven) + - Clone the - [siddhi-map-p4-trpt](https://github.com/cablelabs/siddhi-map-p4-trpt.git) Git project + +Execute following shell commands + +```bash +cd {siddhi-map-p4-trpt directory} +mvn clean compile test +mvn exec:java -Dexec.mainClass=io.siddhi.extension.map.p4.StartSiddhiRuntime \ +-Dexec.args="{siddhi-script-location} {siddhi-script-location} ..." -f pom.xml +``` + +### Kubernetes +Other than a Siddhi runtime server or Java application, another means to deploy +Siddhi scripts is on Kubernetes with the +[siddhi-operator](https://github.com/siddhi-io/siddhi-operator). + +#### Setup +This Ansible [playbook](../../playbooks/siddhi/setup_siddhi_minikube.yml) is an +example of deploying the operators below onto a Minikube instance. + +- Obtain access to a Kubernetes cluster + For testing on a single machine, Please follow the instructions below: + - Setup Docker - [playbook](../../playbooks/general/setup_docker.yml) + - Setup Minikube - [playbook](../../playbooks/general/setup_minikube.yml) + - Start Minikube - [playbook](../../playbooks/general/setup_siddhi_operator.yml) + - Enable ingress addon (/usr/bin/minikube addons enable ingress) +- Install Kafka +```bash +kubectl create namespace kafka +kubectl create -f 'https://strimzi.io/install/latest?namespace=kafka' -n kafka +``` +- Install siddhi-operator +```bash +kubectl apply -f https://github.com/siddhi-io/siddhi-operator/releases/download/v0.2.2/00-prereqs.yaml +kubectl apply -f https://github.com/siddhi-io/siddhi-operator/releases/download/v0.2.2/01-siddhi-operator.yaml +kubectl apply -f https://strimzi.io/examples/latest/kafka/kafka-persistent-single.yaml -n kafka +``` + +#### Deploy AE +To deploy the AE onto Kubernetes, please look at the following example +SiddhiProcess CRDs that can be simply deployed with the standard "kubectl apply -f" +command: +- [UDP to Telemetry Report JSON to Kafka](./kubernetes/trpt-to-kafka.yaml) +- [Telemetry Report JSON to DDoS Detect](./kubernetes/kafka-trpt-ddos-detection.yaml) +- [Telemetry Report JSON to Drop Clear Attack](./kubernetes/kafka-trpt-drop-clear.yaml) diff --git a/docs/ae/kafka-src-drop.png b/docs/ae/kafka-src-drop.png new file mode 100644 index 00000000..7f492b89 Binary files /dev/null and b/docs/ae/kafka-src-drop.png differ diff --git a/docs/ae/kafka-src-pkt.png b/docs/ae/kafka-src-pkt.png new file mode 100644 index 00000000..22540bdd Binary files /dev/null and b/docs/ae/kafka-src-pkt.png differ diff --git a/docs/ae/kubernetes/kafka-trpt-ddos-detection.yaml b/docs/ae/kubernetes/kafka-trpt-ddos-detection.yaml new file mode 100644 index 00000000..5f06aa05 --- /dev/null +++ b/docs/ae/kubernetes/kafka-trpt-ddos-detection.yaml @@ -0,0 +1,92 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example of a TPS-AE SiddhiProcess CRD for reading in Packet +# Telemetry Reports as a JSON string from Kafka and sending out an HTTP POST +# when 10 packets in a 1 second window with the same originating MAC, +# destination IP, and destination port. + +--- +apiVersion: siddhi.io/v1alpha2 +kind: SiddhiProcess +metadata: + name: kafka-trpt-ddos-detect +spec: + apps: + - script: | + @App:name("KafkaSourcePacketJSON") + @source( + type="kafka", + topic.list="trptPacket", + bootstrap.servers="10.110.95.121:9092", + group.id="test", + threading.option="single.thread", + @map( + type="p4-trpt", + @attributes( + src_mac="intHdr.mdStackHdr.origMac", + ip_ver="ipHdr.version", + dst_ip="ipHdr.dstAddr", + dst_port="protoHdr.dstPort" + ) + ) + ) + define stream trptPktStream ( + src_mac string, + ip_ver int, + dst_ip string, + dst_port long); + + @sink( + type="http", + publisher.url="http://localhost:9998/aggAttack", + method="POST", + headers="trp:headers", + @map(type="json") + ) + define stream attackStream ( + src_mac string, + ip_ver int, + dst_ip string, + dst_port long, + count long + ); + + @info(name = "trptJsonQuery") + from trptPktStream#window.time(1 sec) + select src_mac, ip_ver, dst_ip, dst_port, count(ip_ver) as count + group by src_mac, dst_ip, dst_port + having count == 10 + insert into attackStream; + + container: + image: "spisarski/siddhi" + imagePullPolicy: Always + + persistentVolumeClaim: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: standard + volumeMode: Filesystem + + runner: | + state.persistence: + enabled: true + intervalInMin: 5 + revisionsToKeep: 2 + persistenceStore: io.siddhi.distribution.core.persistence.FileSystemPersistenceStore + config: + location: siddhi-app-persistence diff --git a/docs/ae/kubernetes/kafka-trpt-drop-clear.yaml b/docs/ae/kubernetes/kafka-trpt-drop-clear.yaml new file mode 100644 index 00000000..d3d970c5 --- /dev/null +++ b/docs/ae/kubernetes/kafka-trpt-drop-clear.yaml @@ -0,0 +1,88 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example of a TPS-AE SiddhiProcess CRD for reading in Drop +# Telemetry Reports as a JSON string from Kafka and sending out an HTTP DELETE +# when 3 reports with the same hash and count arrive. + +--- +apiVersion: siddhi.io/v1alpha2 +kind: SiddhiProcess +metadata: + name: kafka-trpt-drop-clear +spec: + apps: + - script: | + @App:name("KafkaSourceDropJSON") + @source( + type="kafka", + topic.list="trptDrop", + bootstrap.servers="kafka-service:9092", + group.id="test", + threading.option="single.thread", + @map( + type="p4-trpt", + @attributes( + timestamp="dropHdr.timestamp", + dropKey="dropHdr.dropKey", + dropCount="dropHdr.dropCount" + ) + ) + ) + define stream trptDropStream ( + timestamp long, + dropKey string, + dropCount long + ); + + @sink( + type="http", + publisher.url="http://localhost:9998/aggAttack", + method="DELETE", + headers="trp:headers", + @map(type="json") + ) + define stream dropAttackStream ( + dropKey string, + dropCount long, + count long + ); + + @info(name = "trptJsonQuery") + from trptDropStream#window.time(35 sec) + select dropKey, dropCount, count(dropCount) as count + group by dropKey, dropCount + having count >= 3 + insert into dropAttackStream; + + container: + image: "spisarski/siddhi" + imagePullPolicy: Always + + persistentVolumeClaim: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: standard + volumeMode: Filesystem + + runner: | + state.persistence: + enabled: true + intervalInMin: 5 + revisionsToKeep: 2 + persistenceStore: io.siddhi.distribution.core.persistence.FileSystemPersistenceStore + config: + location: siddhi-app-persistence diff --git a/docs/ae/kubernetes/trpt-to-kafka.yaml b/docs/ae/kubernetes/trpt-to-kafka.yaml new file mode 100644 index 00000000..46b43480 --- /dev/null +++ b/docs/ae/kubernetes/trpt-to-kafka.yaml @@ -0,0 +1,91 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example of a TPS-AE SiddhiProcess CRD for reading in Telemetry +# Report UDP packets, determining the type, sending the JSON String to the +# Kafka topic "trptPacket" for Packet Telemetry Reports created on a P4 switch +# and topic "trptDrop" for Drop Telemetry Reports. + +--- +apiVersion: siddhi.io/v1alpha2 +kind: SiddhiProcess +metadata: + name: trpt-to-kafka +spec: + apps: + - script: | + @App:name("UDPSourceTRPT") + @source( + type="udp", + listen.port="556", + @map( + type="p4-trpt", + @attributes( + in_type="telemRptHdr.inType", + full_json="jsonString" + ) + ) + ) + define stream trptUdpStream (in_type int, full_json object); + + @sink( + type="kafka", + topic="trptPacket", + bootstrap.servers="10.110.95.121:9092", + is.binary.message = "false", + @map(type="text") + ) + define stream trptPacket (full_json object); + + @sink( + type="kafka", + topic="trptDrop", + bootstrap.servers="10.110.95.121:9092", + is.binary.message = "false", + @map(type="text") + ) + define stream trptDrop (full_json object); + + @info(name = "TrptPacket") + from trptUdpStream[in_type != 2] + select full_json + insert into trptPacket; + + @info(name = "TrptDrop") + from trptUdpStream[in_type == 2] + select full_json + insert into trptDrop; + + container: + image: "spisarski/siddhi" + imagePullPolicy: Always + +--- +apiVersion: v1 +kind: Service +metadata: + name: trpt-to-kafka-0 +spec: + type: LoadBalancer + clusterIP: 10.96.100.2 + externalIPs: + - 192.168.86.181 + selector: + siddhi.io/instance: trpt-to-kafka-0 + siddhi.io/name: SiddhiProcess + siddhi.io/part-of: siddhi-operator + siddhi.io/version: 0.2.2 + ports: + - port: 556 + targetPort: 556 + protocol: UDP diff --git a/docs/ae/udp-src-trpt.png b/docs/ae/udp-src-trpt.png new file mode 100644 index 00000000..2d8873fb Binary files /dev/null and b/docs/ae/udp-src-trpt.png differ diff --git a/docs/env_build/CREATE_AUTOMATION_IMAGES.md b/docs/env_build/CREATE_AUTOMATION_IMAGES.md new file mode 100644 index 00000000..eb417150 --- /dev/null +++ b/docs/env_build/CREATE_AUTOMATION_IMAGES.md @@ -0,0 +1,112 @@ +# transparent-security automation for the tofino-model + +The scripts outlined here have been designed to be executed within a CI server + + +### Getting started +Below are the variables used by the env_build CI automation scripts. + +| Variable | Description | Type | Example | +|------------------|-------------------------------------------------------------------------------------------------------------------------------------------|--------|---------------------------------------------------------| +| build_id | This value must be unique to ensure multiple jobs can be run simultaneously from multiple hosts | string | build_id = "example-1" | +| access_key | Amazon EC2 access key | string | access_key = "AKIAIOSFODNN7EXAMPLE" | +| secret_key | Amazon EC2 secret key | string | secret_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" | +| ec2_region | Amazon EC2 region | string | ec2_region = "us-west-2" (default) | +| public_key_file | Used to inject into the VM for SSH access with the user'ubuntu' (defaults to ~/.ssh/id_rsa.pub) | string | public_key_file = "~/.ssh/id_rsa.pub" (default) | +| private_key_file | Used to access the VM via SSH with the user 'ubuntu' (defaults to ~/.ssh/id_rsa) | string | private_key_file = "~/.ssh/id_rsa" (default) | +| env_type | The type of environment being built (only used for creating the environment) | string | env_type = {"tofino"|"siddhi"} (default="tofino") | +| create_ami | When 'yes', the Terraform script will create a new AMI | string | create_ami = "no" (default) | +| bf_sde_version | When this value will be used to obtaining the BF-SDE version | string | bf_sde_version = "9.2.0" (default) | +| bf_sde_profile | The BF-SDE profile name to install | string | bf_sde_profile = "p416_examples_profile" (default) | +| bf_sde_s3_bucket | The S3 bucket in which the BF SDE tar file lives (named bf-sde-{version}.tar) | string | bf_sde_s3_bucket = "EXAMPLE_S3_BUCKET" | + +### Build AMIs for running the tofino-model and Siddhi AE on AWS + +#### Create Tofino BF-SDE EC2 image with Terraform + +##### Install Terraform +https://www.terraform.io/downloads.html + +##### Upload BF-SDE archive +Create s3 bucket and upload the BF-SDE tar/tgz using the same AWS credentials. + +##### Configure +Please see the following example Terraform Variable files to create the files +to hold the Terraform program arguments via the --var-file command line option +in the "Execute" section below. + +- [my-automation-base.tfvars template](../terraform/automation-base.example.tfvars) +- [my-tofino-build.tfvars template](../terraform/tofino-build.example.tfvars) +- [my-siddhi-maven-build.tfvars template](../terraform/siddhi-maven-build.example.tfvars) + +##### Build Tofino image +This step will create an VM on AWS, install all Tofino 9.2.0 dependencies, +then create an AMI. (note: this process will take ~90 min to complete) + +```bash +cd transparent-security/automation/p4/env-build +terraform init +terraform apply -auto-approve -var-file="my-automation-base.tfvars -var-file="my-tofino-build.tfvars" -var build_id={i.e. "my-build-x"}" +``` + +##### Build Siddhi Maven image +This step will create an VM on AWS, install everything required to run Siddhi +with the udp & p4-trpt extensions and Kafka. (note: this image is only required +for the "lab_trial" p4/tofino scenario and will take ~30 min to complete) + +```bash +cd transparent-security/automation/p4/env-build +terraform init +terraform apply -auto-approve -var-file="my-automation-base.tfvars -var-file="my-siddhi-maven-build.tfvars" -var build_id={i.e. "my-build-y"}" +``` + +##### Sample output from the environment build terraform script +Sample Output: + +```bash +. +. +. +Apply complete! Resources: 5 added, 0 changed, 0 destroyed. + +Outputs: + +ami-id = ami-xxx +ip = xx.xx.xx.xx +``` + +Save the ami-ids from each run to be used for running the p4/tofino automation scripts. + +### Remove the AMI for terraform state + +Remove the AMI from terraform state so that it will remain after destroying the +VM should you want to continue to use this image. + +```bash +terraform state rm aws_ami_from_instance.transparent-security-env-build +Removed aws_ami_from_instance.transparent-security-env-build +Successfully removed 1 resource instance(s). +``` + +### Clean up the VM used to create the AMI + +This step will remove everything except the AMI that was used to create the VM +when the call above has been made. + +```bash +terraform destroy -auto-approve -var-file="/path/to/my-tofino.tfvars -var-file="(my-tofino-build.tfvars|my-siddhi-maven-build.tfvars)" +``` + +Sample output: + +```bash +. +. +. +aws_key_pair.transparent-security-mini-pk: Destruction complete after 0s +aws_security_group.transparent-security-img-sg: Destruction complete after 0s + +Destroy complete! Resources: 5 destroyed. + +Process finished with exit code 0 +``` diff --git a/docs/mininet-example.tfvars b/docs/mininet-example.tfvars deleted file mode 100644 index e6e6bc4f..00000000 --- a/docs/mininet-example.tfvars +++ /dev/null @@ -1,27 +0,0 @@ -#Unique ID for this build -build_id = "changeme-1" - -# EC2 credentials -# NOTE-The mininet-ami variable MUST be changed to point to the image created from the CI env-build -# Amazon EC2 access key -access_key = "AKIAIOSFODNN7EXAMPLE" - -# Amazon EC2 secret key -secret_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" - -# Amazon EC2 region; Optional to change -ec2_region = "us-west-2" - -# Used to inject into the VM for SSH access with the user'ubuntu' (defaults to ~/.ssh/id_rsa.pub) -# public_key_file = "~/.ssh/id_rsa.pub" - -# Used to access the VM via SSH with the user 'ubuntu' (defaults to ~/.ssh/id_rsa) -# private_key_file = "~/.ssh/id_rsa" - -# The type of environemnt being used -# Only used for creating the environment -# env_type = "mininet" - -# MUST be changed to the AMI ID generated from CI env-build -# Only used for running simulator -# mininet_ami = "ami-060d055b5ca40de8c" diff --git a/docs/terraform/automation-base.example.tfvars b/docs/terraform/automation-base.example.tfvars new file mode 100644 index 00000000..d799aad0 --- /dev/null +++ b/docs/terraform/automation-base.example.tfvars @@ -0,0 +1,4 @@ +# EC2 credentials +access_key = "myAccessKey" +secret_key = "mySecretKey" +ec2_region = "my-region" diff --git a/docs/terraform/siddhi-maven-build.example.tfvars b/docs/terraform/siddhi-maven-build.example.tfvars new file mode 100644 index 00000000..5a515232 --- /dev/null +++ b/docs/terraform/siddhi-maven-build.example.tfvars @@ -0,0 +1,4 @@ +# EC2 credentials for Transparent Security CI +env_type = "siddhi" +ubuntu_version = "20" +create_ami = "yes" diff --git a/docs/terraform/tofino-build.example.tfvars b/docs/terraform/tofino-build.example.tfvars new file mode 100644 index 00000000..73cca79e --- /dev/null +++ b/docs/terraform/tofino-build.example.tfvars @@ -0,0 +1,4 @@ +# EC2 credentials for Transparent Security CI +bf_sde_s3_bucket = "my-s3-bucket" +env_type = "tofino" +create_ami = "yes" diff --git a/docs/terraform/tofino-int.example.tfvars b/docs/terraform/tofino-int.example.tfvars new file mode 100644 index 00000000..692aaaac --- /dev/null +++ b/docs/terraform/tofino-int.example.tfvars @@ -0,0 +1,7 @@ +tofino = { + sde_version = "9.2.0" + bfrt_ami = "ami-xxx" +} +siddhi_ae_ami = "ami-yyy" +vpc_id = "vpc-xxx" +vpc_subnet_prfx = "xxx.xx" diff --git a/docs/tofino/RUN_CI_AUTOMATION.md b/docs/tofino/RUN_CI_AUTOMATION.md new file mode 100644 index 00000000..dbe4f4aa --- /dev/null +++ b/docs/tofino/RUN_CI_AUTOMATION.md @@ -0,0 +1,89 @@ +# transparent-security automation for the tofino-model + +The scripts outlined here can be executed within a CI server or locally during +development + +### Getting started +Below are the variables used by the tofino CI automation scripts. + +| Variable | Description | Type | Example | +|------------------|-------------------------------------------------------------------------------------------------------------------------------------------|--------|---------------------------------------------------------| +| build_id | This value must be unique to ensure multiple jobs can be run simultaneously from multiple hosts | string | build_id = "example-1" | +| access_key | Amazon EC2 access key | string | access_key = "AKIAIOSFODNN7EXAMPLE" | +| secret_key | Amazon EC2 secret key | string | secret_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" | +| ec2_region | Amazon EC2 region | string | ec2_region = "us-west-2" (default) | +| public_key_file | Used to inject into the VM for SSH access with the user'ubuntu' (defaults to ~/.ssh/id_rsa.pub) | string | public_key_file = "~/.ssh/id_rsa.pub" (default) | +| private_key_file | Used to access the VM via SSH with the user 'ubuntu' (defaults to ~/.ssh/id_rsa) | string | private_key_file = "~/.ssh/id_rsa" (default) | +| bf_sde_version | When this value will be used to obtaining the BF-SDE version | string | bf_sde_version = "9.2.0" (default) | + +### Run terraform to launch the cluster on AWS. + +##### Configure +Please see the following example Terraform Variable files to create the files +to hold the Terraform program arguments via the --var-file command line option +in the "Execute" section below. + +- [my-automation-base.tfvars template](../terraform/automation-base.example.tfvars) +- [my-tofino-int.tfvars template](../terraform/tofino-int.example.tfvars) + + +```bash +cd transparent-security/automation/p4/tofino +terraform init +terraform apply -auto-approve -var-file="my-automation-base.tfvars -var-file="my-tofino-build.tfvars" -var scenario_name=(aggregate|core|lab_trial)" +``` + +### SSH into orchestrator/controller machine + +Use the SSH keys indicated in the variable file to login to the VM. +```bash +# from transparent-security/automation/p4/tofino directory +ssh -i { key file } ubuntu@$(terraform output ip) +``` + +### What is deployed +Controller/Orchestrator node with outside access + +#### scenario_name=lab_trial +* 1 orchestrator/controller node (Performs deployment and runs SDN controller) +* 4 network nodes (Standard Linux VMs) +* 2 Tofino switches (Linux VMs with the BF-SDE running the P4 programs) + +###### Switches (with bf-sde-{version} and transparent-security installed into python runtime) +- core (running core.p4) +- aggregate (running aggregate.p4) + +###### Nodes (vanilla linux with transparent-security installed into python runtime) +- inet (to core) +- ae (to core) +- host1 (to aggregate) +- host2 (to core) + +#### scenario_name=aggregate|core +* 1 orchestrator/controller node +* 2 network nodes +* 1 Tofino switch + +###### Switches (with bf-sde-{version} and transparent-security installed into python runtime) +- runs (aggregate|core).p4 + +###### Nodes (vanilla linux with transparent-security installed into python runtime) +- host1 (southbound server node) +- host2 (northbound server node) + +#### Accessing the switches and nodes from orchestrator +From the orchestrator node, you can gain access to all other nodes and switch VMs +by name with user 'ubuntu': + +```bash +ssh {switch_name | node_name} +``` + +### Cleanup the simulation environment + +This will remove the VM and other artifacts created when it was deployed. + +```bash +# from transparent-security/automation/p4/tofino directory +terraform destroy -auto-approve -var-file="my-automation-base.tfvars -var-file="my-tofino-build.tfvars" -var scenario_name="destroy" +``` diff --git a/docs/tofino/automation.md b/docs/tofino/automation.md deleted file mode 100644 index 8cfe3090..00000000 --- a/docs/tofino/automation.md +++ /dev/null @@ -1,170 +0,0 @@ -# transparent-security automation for the tofino-model - -The scripts outlined here have been designed to be executed within a CI server - - -### Getting started -Please see "Client system setup" in ../BUILD.md - -Please uses these variables instead -Copy the example variable file docs/tofino-example.tfvars to a working -directory and make changes to adapt the file to your local environment. - -| Variable | Description | Type | Example | -|------------------|-------------------------------------------------------------------------------------------------------------------------------------------|--------|---------------------------------------------------------| -| build_id | This value must be unique to ensure multiple jobs can be run simultaneously from multiple hosts | string | build_id = "test-1" | -| access_key | Amazon EC2 access key | string | access_key = "AKIAIOSFODNN7EXAMPLE" | -| secret_key | Amazon EC2 secret key | string | secret_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" | -| ec2_region | Amazon EC2 region | string | ec2_region = "us-west-2" | -| public_key_file | Used to inject into the VM for SSH access with the user'ubuntu' (defaults to ~/.ssh/id_rsa.pub) | string | public_key_file = "~/.ssh/id_rsa.pub" | -| private_key_file | Used to access the VM via SSH with the user 'ubuntu' (defaults to ~/.ssh/id_rsa) | string | private_key_file = "~/.ssh/id_rsa" | -| env_type | The type of environment being built (only used for creating the environment) | string | env_type = "tofino" | - -### Build AMI for running the tofino-model on AWS - -#### Create VM with Terraform - -Create s3 bucket using the same AWS credentials for the BF-SDE tar/tgz -bf_sde_s3_bucket -This step will creat an VM on AWS, install all Tofino 9.2.0 dependencies and create an AMI. - -```bash -cd transparent-security/automation/p4/env-build -terraform init -terraform apply -auto-approve -var-file="/path/to/my-tofino.tfvars -var env_type=tofino" -``` - -Sample Output: - -```bash -aws_key_pair.transparent-security-mini-pk: Creating... -aws_security_group.transparent-security-img-sg: Creating... -aws_key_pair.transparent-security-mini-pk: Creation complete after 1s -. -. -. -Apply complete! Resources: 12 added, 0 changed, 0 destroyed. - -Outputs: - -ami-id = ami-0393652ac3fbc331e -ip = 34.211.114.181 -``` - -Save the ami-id and it to your variables file. - -### Remove the AMI for terraform state - -Remove the AMI from the terraform state so that it will remain after destroying the VM. - -```bash -terraform state rm aws_ami_from_instance.transparent-security-env-build -Removed aws_ami_from_instance.transparent-security-env-build -Successfully removed 1 resource instance(s). -``` - -### Clean up the VM used to create the AMI - -This step will remove everything except the AMI that was used to create the VM. - -```bash -terraform destroy -auto-approve -var-file="/path/to/my-tofino.tfvars" -``` - -Sample output: - -```bash -aws_key_pair.transparent-security-mini-pk: Refreshing state... [id=terraform-20191213203053435500000001] -aws_security_group.transparent-security-img-sg: Refreshing state... [id=sg-057e54e0162c6251a] -. -. -. -Destroy complete! Resources: 4 destroyed. -``` - -## Create a virtual Tofino switch environment (using Terraform) - -Use the environment file created in section 2.4 - -### Run terraform to launch the cluster on AWS. - -Variable file must also contain the following lines as the default is tied to -a CableLabs AWS account: -```hcl-terraform -variable "tofino" { - default = { - sde_version = "9.2.0" - ami = "{{ your ami-version }}" - } -} -``` - -```bash -cd transparent-security/automation/p4/tofino -terraform init -terraform apply -auto-approve -var-file="/path/to/my-tofino.tfvars -var build_id={your unique ID} -var scenario_name=(full|gateway|aggregate|core)" -``` - -### SSH into orchestrator/controller machine - -Use the SSH keys indicated in the variable file to login to the VM. -```bash -# from transparent-security/automation/p4/tofino directory -ssh -i { key file } ubuntu@$(terraform output ip) -``` - -### What is deployed -Controller/Orchestrator node with outside access - -#### scenario_name=full -* 1 orchestrator/controller node (Performs deployment and runs SDN controller) -* 9 network nodes (Standard Linux VMs) -* 5 Tofino switches (Linux VMs with Tofino Model installed running the P4 programs) - -###### Switches (with bf-sde-{version} and transparent-security installed into python runtime) -- core (running core.p4) -- aggregate (running aggregate.p4) -- gateway1 (running gateway.p4) -- gateway2 (running gateway.p4) -- gateway3 (running gateway.p4) - -###### Nodes (vanilla linux with transparent-security installed into python runtime) -- inet (to core) -- ae (to core) -- Camera1 (to gateway1) -- Game1 (to gateway1) -- NAS1 (to gateway1) -- Camera2 (to gateway2) -- Game2 (to gateway2) -- Camera3 (to gateway3) -- Game3 (to gateway3) - -#### scenario_name=gatweay|aggregate|core -* 1 orchestrator/controller node -* 3 network nodes -* 1 Tofino switch - -###### Switches (with bf-sde-{version} and transparent-security installed into python runtime) -- runs {scenario_name}.p4 - -###### Nodes (vanilla linux with transparent-security installed into python runtime) -- host1 (southbound server node) -- host2 (northbound server node) -- clone (northbound server node) - -#### Accessing the switches and nodes from orchestrator -From the orchestrator node, you can gain access to all other nodes and switch VMs -by name with user 'ubuntu': - -```bash -ssh {switch_name | node_name} -``` - -### Cleanup the simulation environment - -This will remove the VM and other artifacts created when it was deployed. - -```bash -# from transparent-security/automation/p4/tofino directory -terraform destroy -auto-approve -var-file="/path/to/my-tofino.tfvars" -var scenario_name="cleanup" -``` diff --git a/examples/kafka-trpt-ddos-detection.yaml b/examples/kafka-trpt-ddos-detection.yaml new file mode 100644 index 00000000..43cbf4bf --- /dev/null +++ b/examples/kafka-trpt-ddos-detection.yaml @@ -0,0 +1,87 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: siddhi.io/v1alpha2 +kind: SiddhiProcess +metadata: + name: kafka-trpt-ddos-detect +spec: + apps: + - script: | + @App:name("KafkaSourcePacketJSON") + @source( + type="kafka", + topic.list="trptPacket", + bootstrap.servers="10.110.95.121:9092", + group.id="test", + threading.option="single.thread", + @map( + type="p4-trpt", + @attributes( + src_mac="intHdr.mdStackHdr.origMac", + ip_ver="ipHdr.version", + dst_ip="ipHdr.dstAddr", + dst_port="protoHdr.dstPort" + ) + ) + ) + define stream trptPktStream ( + src_mac string, + ip_ver int, + dst_ip string, + dst_port long); + + @sink( + type="http", + publisher.url="http://localhost:9998/aggAttack", + method="POST", + headers="trp:headers", + @map(type="json") + ) + define stream attackStream ( + src_mac string, + ip_ver int, + dst_ip string, + dst_port long, + count long + ); + + @info(name = "trptJsonQuery") + from trptPktStream#window.time(1 sec) + select src_mac, ip_ver, dst_ip, dst_port, count(ip_ver) as count + group by src_mac, dst_ip, dst_port + having count == 10 + insert into attackStream; + + container: + image: "spisarski/siddhi" + imagePullPolicy: Always + + persistentVolumeClaim: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: standard + volumeMode: Filesystem + + runner: | + state.persistence: + enabled: true + intervalInMin: 5 + revisionsToKeep: 2 + persistenceStore: io.siddhi.distribution.core.persistence.FileSystemPersistenceStore + config: + location: siddhi-app-persistence diff --git a/examples/kafka-trpt-drop-clear.yaml b/examples/kafka-trpt-drop-clear.yaml new file mode 100644 index 00000000..47788873 --- /dev/null +++ b/examples/kafka-trpt-drop-clear.yaml @@ -0,0 +1,84 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: siddhi.io/v1alpha2 +kind: SiddhiProcess +metadata: + name: kafka-trpt-drop-clear +spec: + apps: + - script: | + @App:name("KafkaSourceDropJSON") + @source( + type="kafka", + topic.list="trptDrop", + bootstrap.servers="kafka-service:9092", + group.id="test", + threading.option="single.thread", + @map( + type="p4-trpt", + @attributes( + timestamp="dropHdr.timestamp", + dropKey="dropHdr.dropKey", + dropCount="dropHdr.dropCount" + ) + ) + ) + define stream trptDropStream ( + timestamp long, + dropKey string, + dropCount long + ); + + @sink( + type="http", + publisher.url="http://localhost:9998/aggAttack", + method="DELETE", + headers="trp:headers", + @map(type="json") + ) + define stream dropAttackStream ( + dropKey string, + dropCount long, + count long + ); + + @info(name = "trptJsonQuery") + from trptDropStream#window.time(35 sec) + select dropKey, dropCount, count(dropCount) as count + group by dropKey, dropCount + having count >= 3 + insert into dropAttackStream; + + container: + image: "spisarski/siddhi" + imagePullPolicy: Always + + persistentVolumeClaim: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: standard + volumeMode: Filesystem + + runner: | + state.persistence: + enabled: true + intervalInMin: 5 + revisionsToKeep: 2 + persistenceStore: io.siddhi.distribution.core.persistence.FileSystemPersistenceStore + config: + location: siddhi-app-persistence diff --git a/examples/kafka.yaml b/examples/kafka.yaml new file mode 100644 index 00000000..59c6a169 --- /dev/null +++ b/examples/kafka.yaml @@ -0,0 +1,123 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: zookeeper-deploy +spec: + replicas: 2 + selector: + matchLabels: + app: zookeeper-1 + template: + metadata: + labels: + app: zookeeper-1 + spec: + containers: + - name: zoo1 +# image: bitnami/zookeeper + image: digitalwonderland/zookeeper +# image: zookeeper +# imagePullPolicy: Always + ports: + - containerPort: 2181 + env: + - name: ZOOKEEPER_ID + value: "1" + - name: ZOOKEEPER_SERVER_1 + value: zoo1 + +--- +apiVersion: v1 +kind: Service +metadata: + name: zoo1 + labels: + app: zookeeper-1 +spec: + ports: + - name: client + port: 2181 + protocol: TCP + - name: follower + port: 2888 + protocol: TCP + - name: leader + port: 3888 + protocol: TCP + selector: + app: zookeeper-1 + +--- +apiVersion: v1 +kind: Service +metadata: + name: kafka-service + labels: + name: kafka +spec: + ports: + - port: 9092 + targetPort: 9092 + protocol: TCP + selector: + app: kafka +# id: "0" + type: LoadBalancer +# clusterIP: 10.96.1.2 + +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: kafka-broker0 +spec: + replicas: 2 + selector: + matchLabels: + app: kafka + id: "0" + template: + metadata: + labels: + app: kafka + id: "0" + spec: + containers: + - name: kafka + image: wurstmeister/kafka +# image: bitnami/kafka + ports: + - containerPort: 9092 + env: + - name: KAFKA_ADVERTISED_PORT + value: "30718" + - name: KAFKA_ADVERTISED_HOST_NAME + value: 10.96.1.2 + - name: KAFKA_ZOOKEEPER_CONNECT + value: zoo1:2181 + - name: KAFKA_BROKER_ID + value: "0" +# - name: KAFKA_CREATE_TOPICS +# value: admintome-test:1:1 +# - name: KAFKA_LISTENERS +# value: LISTENER_PUBLIC://kafka-service:29092,LISTENER_INTERNAL://localhost:9092 +# - name: KAFKA_ADVERTISED_LISTENERS +# value: LISTENER_PUBLIC://kafka-service:29092,LISTENER_INTERNAL://localhost:9092 +# - name: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP +# value: LISTENER_PUBLIC:PLAINTEXT,LISTENER_INTERNAL:PLAINTEXT +# - name: KAFKA_INTER_BROKER_LISTENER_NAME +# value: LISTENER_PUBLIC diff --git a/examples/templates/kafka-trpt-ddos-detection.yaml.j2 b/examples/templates/kafka-trpt-ddos-detection.yaml.j2 new file mode 100644 index 00000000..a8dcb0b1 --- /dev/null +++ b/examples/templates/kafka-trpt-ddos-detection.yaml.j2 @@ -0,0 +1,87 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: siddhi.io/v1alpha2 +kind: SiddhiProcess +metadata: + name: kafka-trpt-ddos-detect +spec: + apps: + - script: | + @App:name("KafkaSourcePacketJSON") + @source( + type="kafka", + topic.list="trptPacket", + bootstrap.servers="{{ kafka_svc_ip }}:9092", + group.id="test", + threading.option="single.thread", + @map( + type="p4-trpt", + @attributes( + src_mac="intHdr.mdStackHdr.origMac", + ip_ver="ipHdr.version", + dst_ip="ipHdr.dstAddr", + dst_port="protoHdr.dstPort" + ) + ) + ) + define stream trptPktStream ( + src_mac string, + ip_ver int, + dst_ip string, + dst_port long); + + @sink( + type="http", + publisher.url="http://{{ sdn_ip }}:9998/aggAttack", + method="POST", + headers="trp:headers", + @map(type="json") + ) + define stream attackStream ( + src_mac string, + ip_ver int, + dst_ip string, + dst_port long, + count long + ); + + @info(name = "trptJsonQuery") + from trptPktStream#window.time(1 sec) + select src_mac, ip_ver, dst_ip, dst_port, count(ip_ver) as count + group by src_mac, dst_ip, dst_port + having count == 10 + insert into attackStream; + + container: + image: "spisarski/siddhi" + imagePullPolicy: Always + + persistentVolumeClaim: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: standard + volumeMode: Filesystem + + runner: | + state.persistence: + enabled: true + intervalInMin: 5 + revisionsToKeep: 2 + persistenceStore: io.siddhi.distribution.core.persistence.FileSystemPersistenceStore + config: + location: siddhi-app-persistence diff --git a/examples/templates/kafka-trpt-drop-clear.yaml.j2 b/examples/templates/kafka-trpt-drop-clear.yaml.j2 new file mode 100644 index 00000000..3f80d227 --- /dev/null +++ b/examples/templates/kafka-trpt-drop-clear.yaml.j2 @@ -0,0 +1,84 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: siddhi.io/v1alpha2 +kind: SiddhiProcess +metadata: + name: kafka-trpt-drop-clear +spec: + apps: + - script: | + @App:name("KafkaSourceDropJSON") + @source( + type="kafka", + topic.list="trptDrop", + bootstrap.servers="{{ kafka_svc_ip }}:9092", + group.id="test", + threading.option="single.thread", + @map( + type="p4-trpt", + @attributes( + timestamp="dropHdr.timestamp", + dropKey="dropHdr.dropKey", + dropCount="dropHdr.dropCount" + ) + ) + ) + define stream trptDropStream ( + timestamp long, + dropKey string, + dropCount long + ); + + @sink( + type="http", + publisher.url="http://{{ sdn_ip }}:9998/aggAttack", + method="DELETE", + headers="trp:headers", + @map(type="json") + ) + define stream dropAttackStream ( + dropKey string, + dropCount long, + count long + ); + + @info(name = "trptJsonQuery") + from trptDropStream#window.time(35 sec) + select dropKey, dropCount, count(dropCount) as count + group by dropKey, dropCount + having count >= 3 + insert into dropAttackStream; + + container: + image: "spisarski/siddhi" + imagePullPolicy: Always + + persistentVolumeClaim: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: standard + volumeMode: Filesystem + + runner: | + state.persistence: + enabled: true + intervalInMin: 5 + revisionsToKeep: 2 + persistenceStore: io.siddhi.distribution.core.persistence.FileSystemPersistenceStore + config: + location: siddhi-app-persistence diff --git a/examples/templates/trpt-pv.yaml.j2 b/examples/templates/trpt-pv.yaml.j2 new file mode 100644 index 00000000..f21afb6f --- /dev/null +++ b/examples/templates/trpt-pv.yaml.j2 @@ -0,0 +1,15 @@ +kind: PersistentVolume +apiVersion: v1 +metadata: + name: siddhi-pv + labels: + type: local +spec: + storageClassName: standard + persistentVolumeReclaimPolicy: Delete + capacity: + storage: 1Gi + accessModes: + - ReadWriteOnce + hostPath: + path: "/home/siddhi_user/" diff --git a/examples/templates/trpt-to-kafka.yaml.j2 b/examples/templates/trpt-to-kafka.yaml.j2 new file mode 100644 index 00000000..8bb5a190 --- /dev/null +++ b/examples/templates/trpt-to-kafka.yaml.j2 @@ -0,0 +1,84 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: siddhi.io/v1alpha2 +kind: SiddhiProcess +metadata: + name: trpt-to-kafka +spec: + apps: + - script: | + @App:name("UDPSourceTRPT") + @source( + type="udp", + listen.port="556", + @map( + type="p4-trpt", + @attributes( + in_type="telemRptHdr.inType", + full_json="jsonString" + ) + ) + ) + define stream trptUdpStream (in_type int, full_json object); + + @sink( + type="kafka", + topic="trptPacket", + bootstrap.servers="{{ kafka_svc_ip }}:9092", + is.binary.message = "false", + @map(type="text") + ) + define stream trptPacket (full_json object); + + @sink( + type="kafka", + topic="trptDrop", + bootstrap.servers="{{ kafka_svc_ip }}:9092", + is.binary.message = "false", + @map(type="text") + ) + define stream trptDrop (full_json object); + + @info(name = "TrptPacket") + from trptUdpStream[in_type != 2] + select full_json + insert into trptPacket; + + @info(name = "TrptDrop") + from trptUdpStream[in_type == 2] + select full_json + insert into trptDrop; + + container: + image: "spisarski/siddhi" + imagePullPolicy: Always + +--- +apiVersion: v1 +kind: Service +metadata: + name: trpt-to-kafka-0 +spec: + type: LoadBalancer + clusterIP: {{ ae_ip }} + selector: + siddhi.io/instance: trpt-to-kafka-0 + siddhi.io/name: SiddhiProcess + siddhi.io/part-of: siddhi-operator + siddhi.io/version: 0.2.2 + ports: + - port: 556 + targetPort: 556 + protocol: UDP diff --git a/examples/test-http.yaml b/examples/test-http.yaml new file mode 100644 index 00000000..035f0ecb --- /dev/null +++ b/examples/test-http.yaml @@ -0,0 +1,36 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hello-app +spec: + selector: + matchLabels: + app: hello + replicas: 1 + template: + metadata: + labels: + app: hello + spec: + containers: + - name: hello + image: "gcr.io/google-samples/hello-app:2.0" + +--- +apiVersion: v1 +kind: Service +metadata: + name: ilb-service + annotations: + networking.gke.io/load-balancer-type: "Internal" + labels: + app: hello +spec: + type: LoadBalancer + selector: + app: hello + ports: + - port: 80 + targetPort: 8080 + protocol: TCP diff --git a/examples/test-networking.yaml b/examples/test-networking.yaml new file mode 100644 index 00000000..e7dbb650 --- /dev/null +++ b/examples/test-networking.yaml @@ -0,0 +1,36 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-networking +spec: + selector: + matchLabels: + app: test-networking + replicas: 1 + template: + metadata: + labels: + app: test-networking + spec: + containers: + - name: ubuntu-test + image: praqma/network-multitool + +--- +apiVersion: v1 +kind: Service +metadata: + name: test-networking-service + annotations: + networking.gke.io/load-balancer-type: "Internal" + labels: + app: test-networking +spec: + type: LoadBalancer + selector: + app: test-networking + ports: + - port: 556 + targetPort: 556 + protocol: UDP diff --git a/examples/trpt-pv.yaml b/examples/trpt-pv.yaml new file mode 100644 index 00000000..f21afb6f --- /dev/null +++ b/examples/trpt-pv.yaml @@ -0,0 +1,15 @@ +kind: PersistentVolume +apiVersion: v1 +metadata: + name: siddhi-pv + labels: + type: local +spec: + storageClassName: standard + persistentVolumeReclaimPolicy: Delete + capacity: + storage: 1Gi + accessModes: + - ReadWriteOnce + hostPath: + path: "/home/siddhi_user/" diff --git a/examples/trpt-to-kafka.yaml b/examples/trpt-to-kafka.yaml new file mode 100644 index 00000000..134a2b95 --- /dev/null +++ b/examples/trpt-to-kafka.yaml @@ -0,0 +1,86 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: siddhi.io/v1alpha2 +kind: SiddhiProcess +metadata: + name: trpt-to-kafka +spec: + apps: + - script: | + @App:name("UDPSourceTRPT") + @source( + type="udp", + listen.port="556", + @map( + type="p4-trpt", + @attributes( + in_type="telemRptHdr.inType", + full_json="jsonString" + ) + ) + ) + define stream trptUdpStream (in_type int, full_json object); + + @sink( + type="kafka", + topic="trptPacket", + bootstrap.servers="10.110.95.121:9092", + is.binary.message = "false", + @map(type="text") + ) + define stream trptPacket (full_json object); + + @sink( + type="kafka", + topic="trptDrop", + bootstrap.servers="10.110.95.121:9092", + is.binary.message = "false", + @map(type="text") + ) + define stream trptDrop (full_json object); + + @info(name = "TrptPacket") + from trptUdpStream[in_type != 2] + select full_json + insert into trptPacket; + + @info(name = "TrptDrop") + from trptUdpStream[in_type == 2] + select full_json + insert into trptDrop; + + container: + image: "spisarski/siddhi" + imagePullPolicy: Always + +--- +apiVersion: v1 +kind: Service +metadata: + name: trpt-to-kafka-0 +spec: + type: LoadBalancer + clusterIP: 10.96.100.2 + externalIPs: + - 192.168.86.181 + selector: + siddhi.io/instance: trpt-to-kafka-0 + siddhi.io/name: SiddhiProcess + siddhi.io/part-of: siddhi-operator + siddhi.io/version: 0.2.2 + ports: + - port: 556 + targetPort: 556 + protocol: UDP diff --git a/p4/aggregate/aggregate.p4 b/p4/aggregate/aggregate.p4 deleted file mode 100755 index 9af53dc1..00000000 --- a/p4/aggregate/aggregate.p4 +++ /dev/null @@ -1,273 +0,0 @@ -/* -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -*/ -/* -*- P4_16 -*- */ -#include - -/* TPS includes */ -#include "../include/tps_consts.p4" -#include "../include/tps_headers.p4" -#include "../include/tps_parser.p4" -#include "../include/tps_checksum.p4" -#include "../include/tps_egress.p4" - -/************************************************************************* -************** I N G R E S S P R O C E S S I N G ******************** -*************************************************************************/ - -control TpsAggIngress(inout headers hdr, - inout metadata meta, - inout standard_metadata_t standard_metadata) { - - counter(MAX_DEVICE_ID, CounterType.packets_and_bytes) forwardedPackets; - - action pack_meta_tcp() { - meta.dst_port = hdr.tcp.dst_port; - } - - action pack_meta_udp() { - meta.dst_port = hdr.udp_int.dst_port; - } - - action pack_meta_ipv4() { - meta.ipv6_addr = 0; - meta.ipv4_addr = hdr.ipv4.dstAddr; - } - - action pack_meta_ipv6() { - meta.ipv4_addr = 0; - meta.ipv6_addr = hdr.ipv6.dstAddr; - } - - action data_drop() { - mark_to_drop(standard_metadata); - } - - table data_drop_t { - key = { - hdr.ethernet.src_mac: exact; - meta.ipv4_addr: exact; - meta.ipv6_addr: exact; - meta.dst_port: exact; - } - actions = { - data_drop; - NoAction; - } - size = TABLE_SIZE; - default_action = NoAction(); - } - - action data_forward(egressSpec_t port) { - standard_metadata.egress_spec = port; - } - - table data_forward_t { - key = { - hdr.ethernet.dst_mac: exact; - } - actions = { - data_forward; - NoAction; - } - size = TABLE_SIZE; - default_action = NoAction(); - } - - action add_switch_id(bit<32> switch_id) { - hdr.int_meta_2.setValid(); - hdr.ipv4.totalLen = hdr.ipv4.totalLen + BYTES_PER_SHIM * INT_SHIM_HOP_SIZE; - hdr.udp_int.len = hdr.udp_int.len + SWITCH_ID_HDR_BYTES; - hdr.ipv6.payload_len = hdr.ipv6.payload_len + BYTES_PER_SHIM * INT_SHIM_HOP_SIZE; - hdr.int_shim.length = hdr.int_shim.length + INT_SHIM_HOP_SIZE; - hdr.int_header.remaining_hop_cnt = hdr.int_header.remaining_hop_cnt - 1; - hdr.int_meta_2.switch_id = switch_id; - } - - table add_switch_id_t { - key = { - hdr.udp_int.dst_port: exact; - } - actions = { - add_switch_id; - NoAction; - } - size = TABLE_SIZE; - default_action = NoAction(); - } - - action data_inspect_packet(bit<32> device, bit<32> switch_id) { - hdr.int_shim.setValid(); - hdr.int_header.setValid(); - hdr.int_meta.setValid(); - - hdr.int_shim.npt = INT_SHIM_NPT_UDP_FULL_WRAP; - hdr.int_shim.type = INT_SHIM_TYPE; - hdr.int_shim.length = INT_SHIM_BASE_SIZE; - - hdr.int_header.ver = INT_VERSION; - hdr.int_header.domain_id = INT_SHIM_DOMAIN_ID; - hdr.int_header.meta_len = INT_META_LEN; - hdr.int_header.instr_bit_0 = TRUE; - hdr.int_header.ds_instr_0 = TRUE; - hdr.int_header.ds_flags_1 = TRUE; - hdr.int_header.remaining_hop_cnt = MAX_HOPS; - - hdr.int_meta.switch_id = switch_id; - hdr.int_meta.orig_mac = hdr.ethernet.src_mac; - - forwardedPackets.count(device); - } - - action data_inspect_packet_ipv4() { - hdr.ipv4.ttl = hdr.ipv4.ttl - 1; - hdr.int_shim.next_proto = hdr.ipv4.protocol; - hdr.ipv4.protocol = TYPE_UDP; - hdr.ipv4.totalLen = hdr.ipv4.totalLen + ((bit<16>)hdr.int_shim.length * BYTES_PER_SHIM * INT_SHIM_HOP_SIZE) + UDP_HDR_BYTES; - } - - action data_inspect_packet_ipv6() { - hdr.int_shim.next_proto = hdr.ipv6.next_hdr_proto; - hdr.ipv6.next_hdr_proto = TYPE_UDP; - hdr.ipv6.payload_len = hdr.ipv6.payload_len + IPV6_HDR_BYTES + ((bit<16>)hdr.int_shim.length * BYTES_PER_SHIM * INT_SHIM_HOP_SIZE) + UDP_HDR_BYTES; - } - - table data_inspection_t { - key = { - hdr.ethernet.src_mac: exact; - } - actions = { - data_inspect_packet; - NoAction; - } - size = TABLE_SIZE; - default_action = NoAction(); - } - - action insert_udp_int_for_udp() { - hdr.udp.setValid(); - hdr.udp.src_port = hdr.udp_int.src_port; - hdr.udp.dst_port = hdr.udp_int.dst_port; - hdr.udp.len = hdr.udp_int.len; - hdr.udp_int.src_port = UDP_INT_SRC_PORT; - hdr.udp_int.dst_port = UDP_INT_DST_PORT; - hdr.udp_int.len = hdr.udp.len + ((bit<16>)hdr.int_shim.length * BYTES_PER_SHIM * INT_SHIM_HOP_SIZE) + UDP_HDR_BYTES; - } - - action insert_udp_int_for_tcp_ipv4() { - hdr.udp_int.setValid(); - hdr.udp_int.src_port = UDP_INT_SRC_PORT; - hdr.udp_int.dst_port = UDP_INT_DST_PORT; - hdr.udp_int.len = hdr.ipv4.totalLen - IPV4_HDR_BYTES; - } - - action insert_udp_int_for_tcp_ipv6() { - hdr.udp_int.setValid(); - hdr.udp_int.src_port = UDP_INT_SRC_PORT; - hdr.udp_int.dst_port = UDP_INT_DST_PORT; - hdr.udp_int.len = hdr.ipv6.payload_len - IPV6_HDR_BYTES; - } - - action control_drop() { - mark_to_drop(standard_metadata);; - } - - action generate_learn_notification() { - digest((bit<32>) 1024, - { hdr.arp.src_mac, - standard_metadata.ingress_port - }); - } - - action arp_flood() { - standard_metadata.mcast_grp = 1; - } - - table arp_flood_t { - key = { - hdr.ethernet.dst_mac: exact; - } - actions = { - arp_flood; - NoAction; - } - default_action = NoAction(); - } - - apply { - if (hdr.arp.isValid()) { - generate_learn_notification(); - if (hdr.arp.opcode == 1) { - arp_flood_t.apply(); - } else if (hdr.arp.opcode == 2) { - data_forward_t.apply(); - } - } else if (standard_metadata.egress_spec != DROP_PORT) { - if (hdr.int_shim.isValid()) { - // Add switch ID into existing INT data - add_switch_id_t.apply(); - } else { - // Pack metadata with original header values for data drop - if (hdr.udp_int.isValid()) { - pack_meta_udp(); - } else if (hdr.tcp.isValid()) { - pack_meta_tcp(); - } - if (hdr.ipv4.isValid()) { - pack_meta_ipv4(); - } else if (hdr.ipv6.isValid()) { - pack_meta_ipv6(); - } - - // Check to see if need to add INT - data_inspection_t.apply(); - - // Add IP & Protocol specific data to new INT data - if (hdr.int_shim.isValid()) { - if (hdr.ipv4.isValid()) { - data_inspect_packet_ipv4(); - if (hdr.udp_int.isValid()) { - insert_udp_int_for_udp(); - } else if (hdr.tcp.isValid()) { - insert_udp_int_for_tcp_ipv4(); - } - } - else if (hdr.ipv6.isValid()) { - data_inspect_packet_ipv6(); - if (hdr.udp_int.isValid()) { - insert_udp_int_for_udp(); - } else if (hdr.tcp.isValid()) { - insert_udp_int_for_tcp_ipv6(); - } - } - } - } - data_forward_t.apply(); - } - data_drop_t.apply(); - } -} - -/************************************************************************* -*********************** S W I T C H ************************************ -*************************************************************************/ - -V1Switch( - TpsAggParser(), - TpsVerifyChecksum(), - TpsAggIngress(), - TpsEgress(), - TpsComputeChecksum(), - TpsAggDeparser() -) main; diff --git a/p4/core/core.p4 b/p4/core/core.p4 deleted file mode 100755 index 6aa3ce52..00000000 --- a/p4/core/core.p4 +++ /dev/null @@ -1,336 +0,0 @@ -/* -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -*/ -/* -*- P4_16 arch -*- */ -#include - -/* TPS includes */ -#include "../include/tps_consts.p4" -#include "../include/tps_headers.p4" -#include "../include/tps_checksum.p4" -#include "../include/tps_parser.p4" -#include "../include/tps_egress.p4" - -const bit<32> INT_CTR_SIZE = 1; - -/************************************************************************* -************** I N G R E S S P R O C E S S I N G ******************** -*************************************************************************/ - -control TpsCoreIngress(inout headers hdr, - inout metadata meta, - inout standard_metadata_t standard_metadata) { - - /** - * Responsible for recirculating a packet after egress processing - */ - action recirculate_packet() { - recirculate(standard_metadata); - } - - /** - * Responsible for cloning a packet as ingressed - */ - action clone_packet_i2e() { - clone3(CloneType.I2E, I2E_CLONE_SESSION_ID, standard_metadata); - } - - /** - * Adds INT data if data_inspection_t table has a match on hdr.ethernet.src_mac - */ - action data_inspect_packet(bit<32> switch_id) { - hdr.int_meta_3.setValid(); - hdr.int_shim.length = hdr.int_shim.length + INT_SHIM_HOP_SIZE; - hdr.ipv4.totalLen = hdr.ipv4.totalLen + (INT_SHIM_HOP_SIZE * BYTES_PER_SHIM); - hdr.udp_int.len = hdr.udp_int.len + (INT_SHIM_HOP_SIZE * BYTES_PER_SHIM); - hdr.int_header.remaining_hop_cnt = hdr.int_header.remaining_hop_cnt - 1; - hdr.int_meta_3.switch_id = switch_id; - } - - table data_inspection_t { - key = { - hdr.udp_int.dst_port: exact; - } - actions = { - data_inspect_packet; - NoAction; - } - size = TABLE_SIZE; - default_action = NoAction(); - } - - /** - * Prepares a packet to be forwarded when data_forward tables have a match on dstAddr - */ - action data_forward(egressSpec_t port) { - standard_metadata.egress_spec = port; - } - - table data_forward_t { - key = { - hdr.ethernet.dst_mac: exact; - } - actions = { - data_forward; - NoAction; - } - size = TABLE_SIZE; - default_action = NoAction(); - } - - action arp_forward(egressSpec_t port) { - standard_metadata.egress_spec = port; - } - - table arp_forward_t { - key = { - hdr.ethernet.dst_mac: exact; - } - actions = { - arp_forward; - NoAction; - } - size = TABLE_SIZE; - default_action = NoAction(); - } - - /** - * Removes INT data from a packet - */ - action clear_int() { - hdr.ipv4.protocol = hdr.int_shim.next_proto; - hdr.ipv6.next_hdr_proto = hdr.int_shim.next_proto; - hdr.ipv4.totalLen = hdr.ipv4.totalLen - ((bit<16>)hdr.int_shim.length * BYTES_PER_SHIM * INT_SHIM_HOP_SIZE) - UDP_HDR_BYTES; - hdr.ipv6.payload_len = hdr.ipv6.payload_len - ((bit<16>)hdr.int_shim.length * BYTES_PER_SHIM * INT_SHIM_HOP_SIZE); - - hdr.udp_int.setInvalid(); - hdr.int_shim.setInvalid(); - hdr.int_header.setInvalid(); - hdr.int_meta_2.setInvalid(); - hdr.int_meta_3.setInvalid(); - hdr.int_meta.setInvalid(); - } - - action generate_learn_notification() { - digest((bit<32>) 1024, - { hdr.arp.src_mac, - standard_metadata.ingress_port - }); - } - - action arp_flood() { - standard_metadata.mcast_grp = 1; - } - - table arp_flood_t { - key = { - hdr.ethernet.dst_mac: exact; - } - actions = { - arp_flood; - NoAction; - } - default_action = NoAction(); - } - - apply { - if (hdr.arp.isValid()) { - generate_learn_notification(); - if (hdr.arp.opcode == 1) { - arp_flood_t.apply(); - } - else if (hdr.arp.opcode == 2) { - arp_forward_t.apply(); - } - } else if (standard_metadata.egress_spec != DROP_PORT) { - if (IS_NORMAL(standard_metadata)) { - // First pass - data_inspection_t.apply(); - recirculate_packet(); - } else if (IS_RECIRCULATED(standard_metadata)) { - // second pass - data_forward_t.apply(); - if (hdr.int_shim.isValid()) { - clone_packet_i2e(); - clear_int(); - } - } - } - } -} - - -/************************************************************************* -**************** E G R E S S P R O C E S S I N G ******************** -*************************************************************************/ - -control TpsCoreEgress(inout headers hdr, - inout metadata meta, - inout standard_metadata_t standard_metadata) { - - register>(INT_CTR_SIZE) trpt_pkts; - - action control_drop() { - mark_to_drop(standard_metadata); - } - - /** - * Restrutures data within INT packet into a Telemetry Report packet type for ipv4 - */ - action init_telem_rpt() { - hdr.trpt_eth.setValid(); - hdr.trpt_udp.setValid(); - hdr.trpt_hdr.setValid(); - - hdr.trpt_hdr.node_id = hdr.int_meta_3.switch_id; - hdr.trpt_hdr.rep_type = TRPT_RPT_TYPE_INT_2; - hdr.trpt_hdr.in_type = TRPT_HDR_IN_TYPE_ETH; - hdr.trpt_hdr.md_len = hdr.int_shim.length; - - hdr.trpt_eth.dst_mac = hdr.ethernet.dst_mac; - hdr.trpt_eth.src_mac = hdr.ethernet.src_mac; - - hdr.trpt_udp.dst_port = TRPT_INT_DST_PORT; - - hdr.trpt_hdr.ver = TRPT_VERSION; - hdr.trpt_hdr.domain_id = TRPT_HDR_DOMAIN_ID; - - hdr.trpt_hdr.sequence_no = 0; - hdr.trpt_hdr.sequence_pad = 0; - - hdr.trpt_hdr.rpt_len = hdr.trpt_hdr.rpt_len + hdr.int_shim.length + 5; /* 5 Reflects TRPT ethernet & udp packets - - /* TODO - determine if counter resets to 0 once it reaches max */ - trpt_pkts.read(hdr.trpt_hdr.sequence_no, INT_CTR_SIZE - 1); - hdr.trpt_hdr.sequence_no = hdr.trpt_hdr.sequence_no + 1; - //trpt_pkts.write(INT_CTR_SIZE - 1, hdr.trpt_hdr.sequence_no); - } - - /* Sets the trpt_eth.in_type for IPv4 */ - action set_telem_rpt_in_type_ipv4() { - hdr.trpt_hdr.in_type = TRPT_HDR_IN_TYPE_IPV4; - } - - /* Sets the trpt_eth.in_type for IPv6 */ - action set_telem_rpt_in_type_ipv6() { - hdr.trpt_hdr.in_type = TRPT_HDR_IN_TYPE_IPV6; - } - - /** - * Restrutures data within INT packet into a Telemetry Report packet type for ipv4 - */ - action setup_telem_rpt_ipv4(ip4Addr_t ae_ip) { - hdr.trpt_ipv4.setValid(); - - hdr.trpt_eth.etherType = TYPE_IPV4; - - hdr.trpt_ipv4.version = 0x4; - hdr.trpt_ipv4.ihl = 0x5; - hdr.trpt_ipv4.ttl = DFLT_IPV4_TTL; - hdr.trpt_ipv4.flags = IPV4_DONT_FRAGMENT; - hdr.trpt_ipv4.protocol = TYPE_UDP; - hdr.trpt_ipv4.srcAddr = hdr.ipv4.srcAddr; - hdr.trpt_ipv4.dstAddr = ae_ip; - hdr.trpt_udp.len = hdr.trpt_ipv4.totalLen - IPV4_HDR_BYTES; - hdr.trpt_ipv4.totalLen = (bit<16>)standard_metadata.packet_length + IPV4_HDR_BYTES + UDP_HDR_BYTES + TRPT_HDR_BASE_BYTES; - } - - /** - * Restrutures data within INT packet into a Telemetry Report packet type for ipv4 - */ - action setup_telem_rpt_ipv6(ip6Addr_t ae_ip) { - hdr.trpt_ipv6.setValid(); - - hdr.trpt_eth.etherType = TYPE_IPV6; - - hdr.trpt_ipv6.next_hdr_proto = TYPE_UDP; - hdr.trpt_ipv6.srcAddr = hdr.ipv6.srcAddr; - hdr.trpt_ipv6.dstAddr = ae_ip; - } - - /** - * Updates the TRPT header length value when the underlying packet is IPv4 - */ - action update_trpt_hdr_len_ipv4() { - hdr.trpt_hdr.rpt_len = hdr.trpt_hdr.rpt_len + 5; - /*hdr.trpt_udp.len = hdr.ipv4.totalLen + IPV4_HDR_BYTES + TRPT_HDR_BYTES + 2;*/ - hdr.trpt_udp.len = hdr.ipv4.totalLen + IPV4_HDR_BYTES + TRPT_HDR_BYTES - 4; - hdr.trpt_ipv4.totalLen = (bit<16>)standard_metadata.packet_length + IPV4_HDR_BYTES + UDP_HDR_BYTES + TRPT_HDR_BASE_BYTES; - } - - /** - * Updates the TRPT header length value when the underlying packet is IPv6 - */ - action update_trpt_hdr_len_ipv6() { - hdr.trpt_hdr.rpt_len = hdr.trpt_hdr.rpt_len + 10; - hdr.trpt_udp.len = hdr.ipv6.payload_len + IPV6_HDR_BYTES + TRPT_HDR_BYTES - ETH_HDR_BYTES; - hdr.trpt_ipv6.payload_len = (bit<16>)standard_metadata.packet_length + IPV6_HDR_BYTES + UDP_HDR_BYTES + TRPT_HDR_BASE_BYTES; - } - - /* TODO - Design table properly, currently just making IPv4 or IPv6 Choices */ - table setup_telemetry_rpt_t { - key = { - hdr.udp_int.dst_port: exact; - } - actions = { - setup_telem_rpt_ipv4; - setup_telem_rpt_ipv6; - control_drop; - } - size = TABLE_SIZE; - default_action = control_drop(); - } - - apply { - if (hdr.arp.isValid()) { - if(IS_REPLICATED(standard_metadata)) { - if (standard_metadata.egress_port == standard_metadata.ingress_port) { - control_drop(); - } - } - } else if (standard_metadata.egress_spec != DROP_PORT) { - if (IS_I2E_CLONE(standard_metadata)) { - init_telem_rpt(); - if (hdr.ipv4.isValid()) { - set_telem_rpt_in_type_ipv4(); - } - if (hdr.ipv6.isValid()) { - set_telem_rpt_in_type_ipv6(); - } - setup_telemetry_rpt_t.apply(); - if (hdr.ipv4.isValid()) { - update_trpt_hdr_len_ipv4(); - } else if (hdr.ipv6.isValid()) { - update_trpt_hdr_len_ipv6(); - } - /* Ensure packet is no larger than TRPT_MAX_BYTES */ - truncate(TRPT_MAX_BYTES); - } - } - } -} - - -/************************************************************************* -*********************** S W I T C H ************************************ -*************************************************************************/ - -V1Switch( - TpsCoreParser(), - TpsVerifyChecksum(), - TpsCoreIngress(), - TpsCoreEgress(), - TpsComputeChecksum(), - TpsDeparser() -) main; diff --git a/p4/gateway/gateway.p4 b/p4/gateway/gateway.p4 deleted file mode 100755 index c2001f6f..00000000 --- a/p4/gateway/gateway.p4 +++ /dev/null @@ -1,360 +0,0 @@ -/* -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -*/ -/* -*- P4_16 -*- */ -#include - -/* TPS includes */ -#include "../include/tps_consts.p4" -#include "../include/tps_headers.p4" -#include "../include/tps_parser.p4" -#include "../include/tps_checksum.p4" -#include "../include/tps_egress.p4" - -/************************************************************************* -************** I N G R E S S P R O C E S S I N G ******************** -*************************************************************************/ - -control TpsGwIngress(inout headers hdr, - inout metadata meta, - inout standard_metadata_t standard_metadata) { - - // TODO/FIXME - so this works for both BMV2 & TOFINO - #ifdef BMV2 - counter(MAX_DEVICE_ID, CounterType.packets_and_bytes) forwardedPackets; - counter(MAX_DEVICE_ID, CounterType.packets_and_bytes) droppedPackets; - #endif - - action data_forward(egressSpec_t port, bit <48> switch_mac) { - standard_metadata.egress_spec = port; - hdr.ethernet.src_mac = switch_mac; - } - - table data_forward_t { - key = { - hdr.ethernet.dst_mac: exact; - } - actions = { - data_forward; - NoAction; - } - size = TABLE_SIZE; - default_action = NoAction(); - } - - action data_inspect_packet(bit<32> device, bit<32> switch_id) { - hdr.udp_int.setValid(); - hdr.int_shim.setValid(); - hdr.int_header.setValid(); - hdr.int_meta.setValid(); - - hdr.udp_int.dst_port = UDP_INT_DST_PORT; - - hdr.int_shim.npt = INT_SHIM_NPT_UDP_FULL_WRAP; - hdr.int_shim.type = INT_SHIM_TYPE; - hdr.int_shim.length = INT_SHIM_BASE_SIZE; - - hdr.int_header.ver = INT_VERSION; - hdr.int_header.domain_id = INT_SHIM_DOMAIN_ID; - hdr.int_header.meta_len = INT_META_LEN; - hdr.int_header.instr_bit_0 = TRUE; - hdr.int_header.ds_instr_0 = TRUE; - hdr.int_header.ds_flags_1 = TRUE; - hdr.int_header.remaining_hop_cnt = MAX_HOPS; - - hdr.int_meta.switch_id = switch_id; - hdr.int_meta.orig_mac = hdr.ethernet.src_mac; - - #ifdef BMV2 - forwardedPackets.count(device); - #endif - } - - action data_inspect_packet_ipv4() { - hdr.ipv4.ttl = hdr.ipv4.ttl - 1; - hdr.int_shim.next_proto = hdr.ipv4.protocol; - hdr.ipv4.protocol = TYPE_UDP; - - #ifdef BMV2 - hdr.ipv4.totalLen = hdr.ipv4.totalLen + ((bit<16>)hdr.int_shim.length * BYTES_PER_SHIM * INT_SHIM_HOP_SIZE) + UDP_HDR_BYTES; - #endif - } - - action data_inspect_packet_ipv6() { - hdr.ipv6.next_hdr_proto = TYPE_UDP; - hdr.int_shim.next_proto = hdr.ipv6.next_hdr_proto; - - #ifdef BMV2 - hdr.ipv6.payload_len = hdr.ipv6.payload_len + IPV6_HDR_BYTES + ((bit<16>)hdr.int_shim.length * BYTES_PER_SHIM * INT_SHIM_HOP_SIZE) + UDP_HDR_BYTES; - #endif - } - - table data_inspection_t { - key = { - hdr.ethernet.src_mac: exact; - } - actions = { - data_inspect_packet; - NoAction; - } - size = TABLE_SIZE; - default_action = NoAction(); - } - - action insert_udp_int_for_udp() { - hdr.udp_int.len = hdr.udp.len + ((bit<16>)hdr.int_shim.length * BYTES_PER_SHIM * INT_SHIM_HOP_SIZE) + UDP_HDR_BYTES; - } - - action insert_udp_int_for_tcp() { - hdr.udp_int.len = ((bit<16>)hdr.int_shim.length * BYTES_PER_SHIM * INT_SHIM_HOP_SIZE) + TCP_HDR_BYTES + UDP_HDR_BYTES; - } - - action data_drop(bit<32> device) { - mark_to_drop(standard_metadata); - #ifdef BMV2 - droppedPackets.count(device); - #endif - } - - table data_drop_udp_ipv4_t { - key = { - hdr.ethernet.src_mac: exact; - hdr.ipv4.dstAddr: exact; - hdr.udp.dst_port: exact; - } - actions = { - data_drop; - NoAction; - } - size = TABLE_SIZE; - default_action = NoAction(); - } - - table data_drop_udp_ipv6_t { - key = { - hdr.ethernet.src_mac: exact; - hdr.ipv6.dstAddr: exact; - hdr.udp.dst_port: exact; - } - actions = { - data_drop; - NoAction; - } - size = TABLE_SIZE; - default_action = NoAction(); - } - - table data_drop_tcp_ipv4_t { - key = { - hdr.ethernet.src_mac: exact; - hdr.ipv4.dstAddr: exact; - hdr.tcp.dst_port: exact; - } - actions = { - data_drop; - NoAction; - } - size = TABLE_SIZE; - default_action = NoAction(); - } - - table data_drop_tcp_ipv6_t { - key = { - hdr.ethernet.src_mac: exact; - hdr.ipv6.dstAddr: exact; - hdr.tcp.dst_port: exact; - } - actions = { - data_drop; - NoAction; - } - size = TABLE_SIZE; - default_action = NoAction(); - } - - action control_drop() { - mark_to_drop(standard_metadata);; - } - - action generate_learn_notification() { - digest((bit<32>) 1024, - { hdr.arp.src_mac, - standard_metadata.ingress_port - }); - } - - action nat_learn_notification() { - digest((bit<32>) 1024, - { hdr.udp.src_port, - hdr.tcp.src_port, - hdr.ipv4.srcAddr - }); - } - - action udp_global_to_local(bit <16> dst_port, bit< 32> ip_dstAddr) { - hdr.udp.dst_port = dst_port; - hdr.ipv4.dstAddr = ip_dstAddr; - } - - table udp_global_to_local_t { - key = { - hdr.udp.dst_port: exact; - hdr.ipv4.dstAddr: lpm; - } - actions = { - udp_global_to_local; - NoAction; - } - default_action = NoAction(); - } - - action tcp_global_to_local(bit <16> dst_port, bit< 32> ip_dstAddr) { - hdr.tcp.dst_port = dst_port; - hdr.ipv4.dstAddr = ip_dstAddr; - } - - table tcp_global_to_local_t { - key = { - hdr.tcp.dst_port: exact; - hdr.ipv4.dstAddr: lpm; - } - actions = { - tcp_global_to_local; - NoAction; - } - default_action = NoAction(); - } - - action udp_local_to_global(bit <16> src_port, bit <32> ip_srcAddr) { - hdr.udp.src_port = src_port; - hdr.ipv4.srcAddr = ip_srcAddr; - } - - table udp_local_to_global_t { - key = { - hdr.udp.src_port: exact; - hdr.ipv4.srcAddr: lpm; - } - actions = { - udp_local_to_global; - NoAction; - } - default_action = NoAction(); - } - - action tcp_local_to_global(bit <16> src_port, bit< 32> ip_srcAddr) { - hdr.tcp.src_port = src_port; - hdr.ipv4.srcAddr = ip_srcAddr; - } - - table tcp_local_to_global_t { - key = { - hdr.tcp.src_port: exact; - hdr.ipv4.srcAddr: lpm; - } - actions = { - tcp_local_to_global; - NoAction; - } - default_action = NoAction(); - } - - action mac_lookup(macAddr_t dst_mac) { - hdr.ethernet.dst_mac = dst_mac; - } - - table mac_lookup_ipv4_t { - key = { - hdr.ipv4.dstAddr: lpm; - } - actions = { - mac_lookup; - NoAction; - } - default_action = NoAction(); - } - - table mac_lookup_ipv6_t { - key = { - hdr.ipv6.dstAddr: lpm; - } - actions = { - mac_lookup; - NoAction; - } - default_action = NoAction(); - } - - apply { - if (hdr.udp.isValid()) { - if (hdr.ipv4.isValid()) { - data_drop_udp_ipv4_t.apply(); - } - if (hdr.ipv6.isValid()) { - data_drop_udp_ipv6_t.apply(); - } - } else if (hdr.tcp.isValid()) { - if (hdr.ipv4.isValid()) { - data_drop_tcp_ipv4_t.apply(); - } - if (hdr.ipv6.isValid()) { - data_drop_tcp_ipv6_t.apply(); - } - } - if (hdr.arp.isValid()) { - generate_learn_notification(); - } - else if (standard_metadata.egress_spec != DROP_PORT) { - data_inspection_t.apply(); - - if (hdr.int_shim.isValid()) { - if (hdr.ipv4.isValid()) { - data_inspect_packet_ipv4(); - } - if (hdr.ipv6.isValid()) { - data_inspect_packet_ipv6(); - } - #ifdef BMV2 - nat_learn_notification(); - if (hdr.udp.isValid()) { - udp_local_to_global_t.apply(); - udp_global_to_local_t.apply(); - insert_udp_int_for_udp(); - } - else if (hdr.tcp.isValid()) { - tcp_local_to_global_t.apply(); - tcp_global_to_local_t.apply(); - insert_udp_int_for_tcp(); - } - #endif - } - mac_lookup_ipv4_t.apply(); - mac_lookup_ipv6_t.apply(); - data_forward_t.apply(); - } - } -} - -/************************************************************************* -*********************** S W I T C H ************************************ -*************************************************************************/ - -V1Switch( - TpsGwParser(), - TpsVerifyChecksum(), - TpsGwIngress(), - TpsEgress(), - TpsComputeChecksum(), - TpsDeparser() -) main; diff --git a/playbooks/dependencies/bmv2.yml b/playbooks/dependencies/bmv2.yml deleted file mode 100644 index e96ebf4a..00000000 --- a/playbooks/dependencies/bmv2.yml +++ /dev/null @@ -1,155 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -- hosts: all - gather_facts: no - tasks: - - name: Update cache before installing python3-pip (apt task does not working here) - become: yes - command: apt update - - - name: install packages for behavioral-model - become: yes - apt: - update_cache: yes - name: - - python3-pip - - git - - autoconf - - automake - - cmake - - libjudy-dev - - libgmp-dev - - libpcap-dev - - libboost-dev - - libboost-test-dev - - libboost-program-options-dev - - libboost-system-dev - - libboost-filesystem-dev - - libboost-thread-dev - - libevent-dev - - libtool - - flex - - bison - - pkg-config - - curl - - make - - g++ - - libreadline-dev - - - name: clone behavioral-model - git: - repo: https://github.com/p4lang/behavioral-model.git - dest: ~/behavioral-model - version: "{{ bm_version }}" - - - name: Install behavioral-model dependencies - command: ./install_deps.sh - args: - chdir: ~/behavioral-model - register: cmd_out - changed_when: cmd_out is not failed - async: 600 - ignore_errors: yes - - - name: Upgrade cffi python package for behavioral-model nmpy - become: yes - pip: - name: - - cffi==1.5.2 - when: cmd_out is failed - - - name: Install behavioral-model dependencies - command: ./install_deps.sh - args: - chdir: ~/behavioral-model - register: second_out - changed_when: second_out is not failed - async: 600 - when: cmd_out is failed - -- import_playbook: p4runtime.yml - -- hosts: all - gather_facts: no - tasks: - - name: autogen behavioral-model - command: ./autogen.sh - args: - chdir: ~/behavioral-model - register: cmd_out - changed_when: cmd_out is not failed - - - name: configure behavioral-model - command: ./configure --enable-debugger --with-pi - args: - chdir: ~/behavioral-model - register: cmd_out - changed_when: cmd_out is not failed - - - name: make behavioral-model - command: make -j4 - args: - chdir: ~/behavioral-model - register: cmd_out - changed_when: cmd_out is not failed - async: 1200 - - - name: install behavioral-model - command: sudo make install - args: - chdir: ~/behavioral-model - register: cmd_out - changed_when: cmd_out is not failed - - - name: ldconfig behavioral-model - command: sudo ldconfig - args: - chdir: ~/behavioral-model - register: cmd_out - changed_when: cmd_out is not failed - - - name: autogen behavioral-model target simple_switch_grpc - command: ./autogen.sh - args: - chdir: ~/behavioral-model/targets/simple_switch_grpc - register: cmd_out - changed_when: cmd_out is not failed - - - name: configure behavioral-model target simple_switch_grpc - command: ./configure --with-pi - args: - chdir: ~/behavioral-model/targets/simple_switch_grpc - register: cmd_out - changed_when: cmd_out is not failed - - - name: make behavioral-model target simple_switch_grpc - command: make -j4 - args: - chdir: ~/behavioral-model/targets/simple_switch_grpc - register: cmd_out - changed_when: cmd_out is not failed - - - name: install behavioral-model simple_switch_grpc - command: sudo make install - args: - chdir: ~/behavioral-model/targets/simple_switch_grpc - register: cmd_out - changed_when: cmd_out is not failed - - - name: ldconfig behavioral-model target simple_switch_grpc - command: sudo ldconfig - args: - chdir: ~/behavioral-model/targets/simple_switch_grpc - register: cmd_out - changed_when: cmd_out is not failed diff --git a/playbooks/dependencies/grpc.yml b/playbooks/dependencies/grpc.yml deleted file mode 100644 index 51f13732..00000000 --- a/playbooks/dependencies/grpc.yml +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -- hosts: all - - gather_facts: no - - environment: - LDFLAGS: "-Wl,-s" - - tasks: - - name: apt update - become: yes - apt: - update_cache: yes - - - name: install packages for grpc - become: yes - apt: - name: - - python3-pip - - git - - autoconf - - automake - - libtool - - curl - - make - - g++ - - - name: pip install cygrpc and upgrade setuptools - become: yes - pip: - name: - - cygrpc - - setuptools>=40.3.0 - when: ubuntu_version == "18" - - - name: clone grpc - git: - repo: https://github.com/grpc/grpc - dest: ~/grpc - version: "{{ grpc_version }}" - force: yes - - - name: build grpc - command: make -j4 - args: - chdir: ~/grpc - register: cmd_out - changed_when: cmd_out is not failed - - - name: configure grpc - command: sudo make install - args: - chdir: ~/grpc - register: cmd_out - changed_when: cmd_out is not failed - - - name: ldconfig grpc - command: sudo ldconfig - args: - chdir: ~/grpc - register: cmd_out - changed_when: cmd_out is not failed - - - name: Copy Python {{ python_version }} site-packages to dist-packages PI - command: "sudo cp -r /home/ubuntu/grpc/src/python/grpcio /usr/local/lib/python{{ python_version }}/dist-packages" diff --git a/playbooks/dependencies/mininet.yml b/playbooks/dependencies/mininet.yml deleted file mode 100644 index 367070ac..00000000 --- a/playbooks/dependencies/mininet.yml +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -- hosts: all - - gather_facts: no - - tasks: - - name: Update cache before installing python3-pip (apt task does not working here) - become: yes - command: apt update - - - name: apt install packages for mininet - become: yes - apt: - update_cache: yes - name: - - python3-pip - - git - - - name: Link /usr/bin/pip to /usr/bin/pip3 - become: yes - command: sudo ln -s /usr/bin/pip3 /usr/bin/pip - - - name: pip install setuptools - become: yes - pip: - name: - - setuptools - - - name: clone mininet - git: - repo: https://github.com/mininet/mininet - dest: ~/mininet - version: master - - - name: install mininet - command: ~/mininet/util/install.sh -nwv - register: cmd_out - changed_when: cmd_out is not failed diff --git a/playbooks/dependencies/p4c.yml b/playbooks/dependencies/p4c.yml deleted file mode 100644 index d06c1714..00000000 --- a/playbooks/dependencies/p4c.yml +++ /dev/null @@ -1,88 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -# Requires playbook - protobuf.yml - -- hosts: all - - gather_facts: no - - tasks: - - name: apt update - become: yes - apt: - update_cache: yes - - - name: install packages for p4c - become: yes - apt: - name: - - git - - python3-pip - - g++ - - automake - - libtool - - libgc-dev - - cmake - - flex - - bison - - libfl-dev - - libgmp-dev - - libboost-dev - - libboost-iostreams-dev - - libboost-graph-dev - - llvm - - pkg-config - - tcpdump - - doxygen - - - name: pip install - become: yes - pip: - name: - - scapy - - ipaddr - - ply - - - name: clone p4c - git: - repo: https://github.com/p4lang/p4c - dest: ~/p4c - recursive: yes - version: "{{ p4c_version }}" - - - name: Create p4c build directory - file: - path: ~/p4c/build - state: directory - - - name: cmake p4c - command: cmake .. -DCMAKE_BUILD_TYPE=DEBUG -DENABLE_EBPF=OFF -DENABLE_BMV2=ON - args: - chdir: ~/p4c/build - register: cmd_out - changed_when: cmd_out is not failed - - - name: make -j4 - command: make -j4 - args: - chdir: ~/p4c/build - register: cmd_out - changed_when: cmd_out is not failed - - - name: install p4c - command: sudo make install - args: - chdir: ~/p4c/build - register: cmd_out - changed_when: cmd_out is not failed diff --git a/playbooks/dependencies/p4runtime.yml b/playbooks/dependencies/p4runtime.yml deleted file mode 100644 index 68db123c..00000000 --- a/playbooks/dependencies/p4runtime.yml +++ /dev/null @@ -1,85 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -- hosts: all - - gather_facts: no - - environment: - LDFLAGS: "-Wl,-s" - - tasks: - - name: install packages for p4runtime (PI) - become: yes - apt: - name: - - git - - autoconf - - automake - - libtool - - curl - - make - - g++ - - libjudy-dev - - pkg-config - - libboost-all-dev - - - name: clone PI - git: - repo: https://github.com/p4lang/PI - dest: ~/PI - version: "{{ pi_version }}" - recursive: yes - force: yes - - - name: autogen PI - command: ./autogen.sh - args: - chdir: ~/PI - register: cmd_out - changed_when: cmd_out is not failed - - - name: configure PI - command: ./configure --with-proto - args: - chdir: ~/PI - register: cmd_out - changed_when: cmd_out is not failed - - - name: make PI - command: make -j4 - args: - chdir: ~/PI - register: cmd_out - changed_when: cmd_out is not failed - - - name: install PI - command: sudo make install - args: - chdir: ~/PI - register: cmd_out - changed_when: cmd_out is not failed - - - name: ldconfig PI - command: sudo ldconfig - args: - chdir: ~/PI - register: cmd_out - changed_when: cmd_out is not failed - - - name: Copy Python {{ python_version }} site-packages to dist-packages PI - command: "sudo cp -r /home/ubuntu/PI/proto/py_out/{{ item }} /usr/local/lib/python{{ python_version }}/dist-packages" - with_items: - - p4 - - gnmi - - google diff --git a/playbooks/dependencies/protobuf.yml b/playbooks/dependencies/protobuf.yml deleted file mode 100644 index bfba31a4..00000000 --- a/playbooks/dependencies/protobuf.yml +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -- hosts: all - - gather_facts: no - - environment: - CFLAGS: "-Os" - CXXFLAGS: "-Os" - LDFLAGS: "-Wl,-s" - - tasks: - - name: apt update - become: yes - apt: - update_cache: yes - - - name: install packages for protobuf - become: yes - apt: - name: - - git - - autoconf - - automake - - libtool - - curl - - make - - gcc - - unzip - - - name: clone protobuf - git: - repo: https://github.com/protocolbuffers/protobuf - dest: ~/protobuf - version: "{{ protobuf_version }}" - recursive: yes - force: yes - - - name: autogen protobuf - command: ./autogen.sh - args: - chdir: ~/protobuf - register: cmd_out - changed_when: cmd_out is not failed - - - name: configure protobuf - command: ./configure --prefix=/usr - args: - chdir: ~/protobuf - register: cmd_out - changed_when: cmd_out is not failed - - - name: make protobuf - command: make -j4 - args: - chdir: ~/protobuf - register: cmd_out - changed_when: cmd_out is not failed - - - name: make install protobuf - command: sudo make install - args: - chdir: ~/protobuf - register: cmd_out - changed_when: cmd_out is not failed - - - name: ldconfig protobuf - command: sudo ldconfig - args: - chdir: ~/protobuf - register: cmd_out - changed_when: cmd_out is not failed - - - name: Install protobuf into python - shell: "cd /home/ubuntu/protobuf/python/; sudo python setup.py install" - - - name: Copy Python {{ python_version }} protobuf to dist-packages PI - command: "sudo cp -r /home/ubuntu/protobuf/python/{{ item }} /usr/local/lib/python{{ python_version }}/dist-packages" - with_items: - - google diff --git a/playbooks/env-build/ae/env_build.yml b/playbooks/env-build/ae/env_build.yml deleted file mode 100644 index f3c13b1d..00000000 --- a/playbooks/env-build/ae/env_build.yml +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# https://github.com/p4lang/tutorials/blob/master/vm/user-bootstrap.sh -# Project and script derived in part from the script in the link above ---- -- import_playbook: ../../hcp/ae_node_build.yml diff --git a/playbooks/env-build/mininet/env_build.yml b/playbooks/env-build/mininet/env_build.yml deleted file mode 100644 index aef045c1..00000000 --- a/playbooks/env-build/mininet/env_build.yml +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# https://github.com/p4lang/tutorials/blob/master/vm/user-bootstrap.sh -# Project and script derived in part from the script in the link above ---- -- import_playbook: ../../dependencies/mininet.yml -- import_playbook: ../../dependencies/protobuf.yml -- import_playbook: ../../dependencies/grpc.yml -- import_playbook: ../../dependencies/bmv2.yml -- import_playbook: ../../dependencies/p4c.yml -- import_playbook: ../../general/p4_runtime_shell.yml -- import_playbook: ../../general/final_env_setup.yml diff --git a/playbooks/env-build/siddhi/env_build.yml b/playbooks/env-build/siddhi/env_build.yml index a53ed116..69511097 100644 --- a/playbooks/env-build/siddhi/env_build.yml +++ b/playbooks/env-build/siddhi/env_build.yml @@ -14,4 +14,5 @@ # https://github.com/p4lang/tutorials/blob/master/vm/user-bootstrap.sh # Project and script derived in part from the script in the link above --- -- import_playbook: ../../siddhi/setup.yml +#- import_playbook: ../../siddhi/setup_siddhi_maven.yml +- import_playbook: ../../siddhi/setup_siddhi_minikube.yml diff --git a/playbooks/general/final_env_setup.yml b/playbooks/general/final_env_setup.yml index b33e6864..bc7159a5 100644 --- a/playbooks/general/final_env_setup.yml +++ b/playbooks/general/final_env_setup.yml @@ -11,8 +11,6 @@ # See the License for the specific language governing permissions and # limitations under the License. --- -# Requires playbook - protobuf.yml - - hosts: all gather_facts: no @@ -21,6 +19,7 @@ - name: Install python3-pip and wireshark become: yes apt: + update_cache: yes name: - python3-pip - wireshark-qt @@ -36,8 +35,24 @@ src: ../../requirements.txt dest: ~/tps-py-requirements.txt - - name: Install TPS Python requirements + - name: Install TPS pip requirements command: sudo pip install -r ~/tps-py-requirements.txt + register: pip_install + ignore_errors: yes + + - name: Install TPS pip3 requirements + command: sudo pip3 install -r ~/tps-py-requirements.txt + when: pip_install is failed + + - name: Final upgrade of all apt packages + become: yes + apt: + upgrade: yes + + - name: Final apt cache update + become: yes + apt: + update_cache: yes - name: Disable unattended-upgrades become: yes diff --git a/playbooks/general/setup_docker.yml b/playbooks/general/setup_docker.yml new file mode 100644 index 00000000..a171583c --- /dev/null +++ b/playbooks/general/setup_docker.yml @@ -0,0 +1,73 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This playbook is capable of setting up Docker on deb Linux machines. + +--- +- hosts: "{{ hosts | default('all') }}" + gather_facts: no + become: yes + tasks: + - name: Remove apt dependecies for Docker + apt: + name: + - docker + - docker-engine + - docker.io + - containerd + - runc + state: absent + + - name: apt update + apt: + update_cache: yes + + - name: Install apt dependencies for Docker + apt: + name: + - apt-transport-https + - ca-certificates + - curl + - gnupg + - lsb-release + register: apt_rc + retries: 3 + delay: 10 + until: apt_rc is not failed + + - name: Add Docker's GPG key + shell: curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg + + - name: Add Docker apt repo + shell: | + echo \ + "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + + - name: apt update + apt: + update_cache: yes + + - name: Install apt dependencies for Docker + apt: + name: + - docker-ce + - docker-ce-cli + - containerd.io + register: apt_rc + retries: 3 + delay: 10 + until: apt_rc is not failed + + - name: Add user {{ ansible_user }} to docker group + command: "usermod -aG docker {{ ansible_user }}" diff --git a/playbooks/siddhi/setup_jdk.yml b/playbooks/general/setup_jdk.yml similarity index 100% rename from playbooks/siddhi/setup_jdk.yml rename to playbooks/general/setup_jdk.yml diff --git a/playbooks/siddhi/setup_kafka.yml b/playbooks/general/setup_kafka.yml similarity index 100% rename from playbooks/siddhi/setup_kafka.yml rename to playbooks/general/setup_kafka.yml diff --git a/playbooks/general/setup_minikube.yml b/playbooks/general/setup_minikube.yml new file mode 100644 index 00000000..d5f60463 --- /dev/null +++ b/playbooks/general/setup_minikube.yml @@ -0,0 +1,61 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This playbook is capable of setting Minikube and depends on setup_docker.yml. + +--- +- import_playbook: setup_docker.yml + +- hosts: "{{ hosts | default('all') }}" + gather_facts: no + tasks: + - name: Create minikube binary directory + become: yes + file: + path: /etc/minikube + state: directory + + - name: Get minikube + become: yes + get_url: + url: https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64 + dest: /etc/minikube/minikube-linux-amd64 + + - name: Install minikube + become: yes + command: install minikube-linux-amd64 /usr/bin/minikube + args: + chdir: /etc/minikube + + - name: Install kubectl + become: yes + command: snap install kubectl --classic + + - name: Start minikube to ensure it will work + shell: sudo -u $USER /usr/bin/minikube start --memory=4096 + register: minkube_start_rc + async: 180 + retries: 2 + delay: 5 + until: minkube_start_rc is not failed + + - name: Check Minikube K8s cluster is operational + command: kubectl get po -A + register: kube_pods_out + + - name: Show pods + debug: + var: kube_pods_out + + - name: Stop minikube + shell: sudo -u $USER /usr/bin/minikube stop diff --git a/playbooks/general/setup_siddhi_operator.yml b/playbooks/general/setup_siddhi_operator.yml new file mode 100644 index 00000000..665fee63 --- /dev/null +++ b/playbooks/general/setup_siddhi_operator.yml @@ -0,0 +1,85 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This playbook is responsible for starting minikube, adding the ingress addon, +# setting up Kafka as well as the siddhi-operator which makes it possible to +# deploy CRD's of kind "SiddhiProcess" to K8s. +# Please note that Minikube is not required to install the either of the +# operators (Strimzi Kafka or siddhi-operator) + +--- +- import_playbook: setup_minikube.yml + +- hosts: "{{ hosts | default('all') }}" + gather_facts: no + tasks: + - name: Start minikube to install siddhi-operator + shell: sudo -u $USER /usr/bin/minikube start --memory=4096 + register: minkube_start_rc + async: 180 + retries: 2 + delay: 5 + until: minkube_start_rc is not failed + + - name: Enable minikube ingress addon + command: sudo -u $USER /usr/bin/minikube addons enable ingress + register: ing_out + async: 60 + retries: 2 + delay: 5 + until: ing_out is not failed + + - name: Create K8s namespace "kafka" + command: kubectl create namespace kafka + + - name: Create Strimzi Kafka ClusterRoles, ClusterRoleBindings & other CRDs + command: kubectl create -f 'https://strimzi.io/install/latest?namespace=kafka' -n kafka + + - name: Install Siddhi K8s and Strimzi Kafka operator (see https://strimzi.io/quickstarts/) + command: "kubectl apply -f {{ item }}" + loop: + - https://github.com/siddhi-io/siddhi-operator/releases/download/v0.2.2/00-prereqs.yaml + - https://github.com/siddhi-io/siddhi-operator/releases/download/v0.2.2/01-siddhi-operator.yaml + - https://strimzi.io/examples/latest/kafka/kafka-persistent-single.yaml -n kafka + + - name: Wait for Strimzi Kafka operator + command: kubectl wait kafka/my-cluster --for=condition=Ready --timeout=300s -n kafka + + - name: Get minikube ip + command: sudo -u $USER /usr/bin/minikube ip + register: ip_out + retries: 2 + delay: 5 + until: ip_out is not failed + + - name: minikube IP + debug: + var: ip_out + + - name: Setup minikube/siddhi IP into /etc/hosts + become: yes + shell: "echo ' {{ ip_out.stdout_lines[0] }} siddhi' >> /etc/hosts" + + - name: Check siddhi-operator is Running + command: kubectl get po + register: kube_pods_out + retries: 10 + delay: 5 + until: kube_pods_out.stdout.find("Running") != -1 + + - name: Show pods + debug: + var: kube_pods_out.stdout_lines + + - name: Stop minikube + shell: sudo -u $USER /usr/bin/minikube stop diff --git a/playbooks/general/setup_source.yml b/playbooks/general/setup_source.yml index ae21c9e7..5b5baef4 100644 --- a/playbooks/general/setup_source.yml +++ b/playbooks/general/setup_source.yml @@ -20,48 +20,51 @@ dest_dir: "/home/{{ ansible_user }}/transparent-security" run_tests: "{{ python_unit_tests | default(false) }}" install_python: "{{ install_tps_python | default(true) }}" + install_tps_dependencies: "{{ install_dependencies | default(true) }}" tasks: - - name: Install apt dependencies - apt: - update_cache: yes - name: - - python3-pip - - arping - - iperf3 - register: apt_rc - retries: 3 - delay: 10 - until: apt_rc is not failed - when: ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu' + - block: + - name: Install apt dependencies + apt: + update_cache: yes + name: + - python3-pip + - arping + - iperf3 + register: apt_rc + retries: 3 + delay: 10 + until: apt_rc is not failed + when: ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu' - - name: Install yum dependencies - yum: - update_cache: yes - name: - - python3-pip - - python3-devel - - gcc-c++ - - tcpdump - - net-tools - - iputils - - iperf3 - register: apt_rc - retries: 3 - delay: 10 - until: apt_rc is not failed - when: ansible_distribution == 'CentOS' or ansible_distribution == 'Red Hat Enterprise Linux' + - name: Install yum dependencies + yum: + update_cache: yes + name: + - python3-pip + - python3-devel + - gcc-c++ + - tcpdump + - net-tools + - iputils + - iperf3 + register: apt_rc + retries: 3 + delay: 10 + until: apt_rc is not failed + when: ansible_distribution == 'CentOS' or ansible_distribution == 'Red Hat Enterprise Linux' - - name: Downgrade pip3 scapy version to 2.4.3 due to 2.4.4 bug when running receive_packets.py on centos - pip: - name: - - scapy==2.4.3 - executable: pip3 - register: pip_rc - retries: 3 - delay: 10 - until: pip_rc is not failed - when: ansible_distribution == 'CentOS' or ansible_distribution == 'Red Hat Enterprise Linux' + - name: Downgrade pip3 scapy version to 2.4.3 due to 2.4.4 bug when running receive_packets.py on centos + pip: + name: + - scapy==2.4.3 + executable: pip3 + register: pip_rc + retries: 3 + delay: 10 + until: pip_rc is not failed + when: ansible_distribution == 'CentOS' or ansible_distribution == 'Red Hat Enterprise Linux' + when: install_tps_dependencies | bool - name: Copy local transparent-security source become: no diff --git a/playbooks/general/templates/mininet_service.sh.j2 b/playbooks/general/templates/mininet_service.sh.j2 deleted file mode 100644 index 9948c4a6..00000000 --- a/playbooks/general/templates/mininet_service.sh.j2 +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash - -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Stops then starts mininet with the given topology -mn -c; {{ trans_sec_dir }}/bin/run_p4_mininet.py -t {{ topo_file_loc }} -l {{ log_dir }} diff --git a/playbooks/general/templates/sdn_controller.sh.j2 b/playbooks/general/templates/sdn_controller.sh.j2 index 9979dd5d..059fd1e2 100644 --- a/playbooks/general/templates/sdn_controller.sh.j2 +++ b/playbooks/general/templates/sdn_controller.sh.j2 @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# Runs the SDN controller for Mininet simulation +# Runs the SDN controller {{ trans_sec_dir }}/bin/sdn_controller.py -l {{ log_level }} \ -ainv {{ ansible_inventory_file }} \ -u {{ ansible_user }} \ diff --git a/playbooks/hcp/ae_node_build.yml b/playbooks/hcp/ae_node_build.yml deleted file mode 100644 index c5737b7d..00000000 --- a/playbooks/hcp/ae_node_build.yml +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -#Import playbook -- import_playbook: setup_ae_centos.yml diff --git a/playbooks/hcp/conf_ae_elk.yml b/playbooks/hcp/conf_ae_elk.yml deleted file mode 100644 index acd1ef6e..00000000 --- a/playbooks/hcp/conf_ae_elk.yml +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright (c) 2020 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -- hosts: ae - gather_facts: no - tasks: - - name: "Validate if a DNS tps.sdn.org is present in /etc/hosts file" - become: yes - tags: sdnDnsName - lineinfile: - path: /etc/hosts - regexp: "tps.sdn.org" - line: "tps.sdn.org {{ sdn_ip }}" - state: present - backup: yes - register: sdnDnsNameOut - - - name: POST call to create Index Pattern - uri: - url: "http://localhost:5601/api/saved_objects/index-pattern/packets-*" - method: POST - headers: - kbn-xsrf: "true" - Content-Type: "application/json" - return_content: yes - status_code: 200 - body: "{{ lookup('file','./templates/index_pattern_request_body.json') }}" - body_format: json - register: index_pattern_response diff --git a/playbooks/hcp/delete_ae_history.yml b/playbooks/hcp/delete_ae_history.yml deleted file mode 100644 index 31f6a4c0..00000000 --- a/playbooks/hcp/delete_ae_history.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright (c) 2020 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- hosts: ae - gather_facts: no - tasks: - - name: "Remove all packet indices" - become: yes - uri: - url: "http://localhost:9200/packets-*" - method: DELETE - headers: - kbn-xsrf: "true" - Content-Type: "application/json" - return_content: yes - status_code: 200 - register: delete_packets_index_response diff --git a/playbooks/hcp/setup_ae_centos.yml b/playbooks/hcp/setup_ae_centos.yml deleted file mode 100644 index 4ec76380..00000000 --- a/playbooks/hcp/setup_ae_centos.yml +++ /dev/null @@ -1,242 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -#Import playbook - -- hosts: all - gather_facts: no - vars: - USER_HOME: "/home/centos" - CMAKE_DIR: "{{ USER_HOME }}/cmake-3.5.2" - WIRESHARK_DIR: "{{ USER_HOME }}/wireshark-3.2.6" - PYTHON_DIR: "{{ USER_HOME }}/Python-3.8.5" - tasks: - - name: Uninstall python3-pip & python-pip - become: yes - yum: - name: - - python3-pip - - python-pip - state: absent - - - name: Install yum dependencies - become: yes - yum: - name: - - tcpdump - - vim - - wget - - unzip - - java-1.8.0-openjdk-devel - - git - - gcc - - gcc-c++ - - bison - - flex - - libpcap-devel - - qt-devel - - gtk3-devel - - rpm-build - - libtool - - c-ares-devel - - qt5-qtbase-devel - - qt5-qtmultimedia-devel - - qt5-linguist - - desktop-file-utils - - openssl-devel - - bzip2-devel - - libffi-devel - - libgcrypt-devel - - ncurses-devel - - - name: Get Wireshark v3.2.6 tar - get_url: - url: https://www.wireshark.org/download/src/all-versions/wireshark-3.2.6.tar.xz - dest: "{{ USER_HOME }}" - - - name: Create directory for Wireshark - file: - path: "{{ WIRESHARK_DIR }}" - state: directory - - - name: Extract Wireshark - unarchive: - src: "{{ USER_HOME }}/wireshark-3.2.6.tar.xz" - dest: "{{ USER_HOME }}" - remote_src: yes - - - name: Run rpm-setup script in Wireshark repository to install unmet dependencies - become: yes - command: "{{ WIRESHARK_DIR }}/tools/rpm-setup.sh -y" - - - name: Retrieve cmake tarball - get_url: - url: https://cmake.org/files/v3.5/cmake-3.5.2.tar.gz - dest: "{{ USER_HOME }}" - - - name: Create directory for cmake - file: - path: "{{ CMAKE_DIR }}" - state: directory - - - name: Extract cmake - unarchive: - src: "{{ USER_HOME }}/cmake-3.5.2.tar.gz" - dest: "{{ USER_HOME }}" - remote_src: yes - - - name: Bootstrap prefix usr before executing make for cmake - command: ./bootstrap --prefix=/usr - args: - chdir: "{{ CMAKE_DIR }}" - - - name: Run cmake make install - become: yes - command: make -j4 install - args: - chdir: "{{ CMAKE_DIR }}" - - - name: Verify cmake installed - command: cmake --version - register: cmake_out - args: - chdir: "{{ CMAKE_DIR }}" - - - name: Fail when cmake version is incorrect - fail: - msg: "cmake version is incorrect {{ cmake_out }}" - when: cmake_out.stdout is not search('3.5.2') - - - name: Create build directory for Wireshark installation - file: - path: "{{ WIRESHARK_DIR }}/build" - state: directory - - - name: Run CMAKE command for Wireshark installation - command: "cmake {{ WIRESHARK_DIR }}" - args: - chdir: "{{ WIRESHARK_DIR }}/build" - - - name: Run make Wireshark - command: make -j4 - args: - chdir: "{{ WIRESHARK_DIR }}/build" - - - name: Install Wireshark - become: yes - command: make -j4 install - args: - chdir: "{{ WIRESHARK_DIR }}/build" - - - name: Download and install a specific version of Python v3.8.5 - get_url: - url: https://www.python.org/ftp/python/3.8.5/Python-3.8.5.tgz - dest: "{{ USER_HOME }}" - - - name: Create directory for Python - file: - path: "{{ PYTHON_DIR }}" - state: directory - - - name: Extract Python-3.8.5 - unarchive: - src: "{{ USER_HOME }}/Python-3.8.5.tgz" - dest: "{{ USER_HOME }}" - remote_src: yes - - - name: Configure Python optimizations for Python 3.8 install - command: ./configure --enable-optimizations - args: - chdir: "{{ PYTHON_DIR }}" - - - name: Run INSTALL command for python 3.8 installation - become: yes - command: make -j4 altinstall - args: - chdir: "{{ PYTHON_DIR }}" - - - name: Remove previously existing python3 configuration - become: yes - file: - path: /usr/bin/python3 - state: absent - - - name: Reset Python3 version to Python3.8 installation - become: yes - shell: ln -s /usr/local/bin/python3.8 /usr/bin/python3 - - - name: Get repo OpenDistro Elasticsearch via RPM package - become: yes - get_url: - url: https://d3g5vo6xdbdb9a.cloudfront.net/yum/opendistroforelasticsearch-artifacts.repo - dest: /etc/yum.repos.d - - - name: Install OpenDistro Elasticsearch - become: yes - yum: - name: opendistroforelasticsearch-1.11.0-1 - - - name: Remove link for OpenDistro Elasticsearch installation - become: yes - file: - path: /usr/share/elasticsearch/lib/tools.jar - state: absent - - - name: Add link for OpenDistro Elasticsearch installation - become: yes - command: ln -s /usr/lib/jvm/java-1.8.0/lib/tools.jar /usr/share/elasticsearch/lib/tools.jar - - - name: Remove opendistro_security plugin for current OpenDistro Elasticsearch installation - become: yes - command: /usr/share/elasticsearch/bin/elasticsearch-plugin remove opendistro_security - - - name: Update OpenDistro Elasticsearch config file to disable security - become: yes - copy: - src: ./templates/opendistro_elasticsearch.sh - dest: /etc/elasticsearch/elasticsearch.yml - force: yes - - - name: Install OpenDistro Kibana - become: yes - yum: - name: opendistroforelasticsearch-kibana - - - name: Remove opendistro_security plugin for current OpenDistro Kibana installation - become: yes - command: /usr/share/kibana/bin/kibana-plugin remove opendistro_security --allow-root - - - name: Update OpenDistro Kibana config file - become: yes - copy: - src: ./templates/opendistro_kibana.sh - dest: /etc/kibana/kibana.yml - force: yes - - - name: Get Espcap repository - git: - repo: https://github.com/vichargrave/espcap - dest: "{{ USER_HOME }}/espcap" - - - name: Install requirements for Espcap Elasticsearch v7.x - become: yes - command: "pip3 install -r {{ USER_HOME }}/espcap/requirements-7.x.txt" - - - name: Copy TPS requirements.txt - copy: - src: ../../requirements.txt - dest: "{{ USER_HOME }}/tps-py-requirements.txt" - - - name: Install TPS Python requirements - become: yes - command: "pip3 install -r {{ USER_HOME }}/tps-py-requirements.txt" diff --git a/playbooks/hcp/setup_ae_tcp_pipeline.yml b/playbooks/hcp/setup_ae_tcp_pipeline.yml deleted file mode 100644 index c1f285da..00000000 --- a/playbooks/hcp/setup_ae_tcp_pipeline.yml +++ /dev/null @@ -1,165 +0,0 @@ -# Copyright (c) 2020 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#Import playbook -- import_playbook: setup_ae_elk.yml - -- hosts: ae - gather_facts: no - tasks: - - name: POST call to create TCP data mapping template for incoming data in Elasticsearch - uri: - url: "http://localhost:9200/_template/packets" - method: PUT - return_content: yes - body: "{{ lookup('file','./templates/tcp/tcp_data_mapping.json') }}" - body_format: json - register: tcp_template_response - - - name: POST call to create data parsing pipeline for TCP v4 and UDP v6 for incoming data in Elasticsearch - uri: - url: "http://localhost:9200/_ingest/pipeline/ts_tcp_parsing" - method: PUT - return_content: yes - body: "{{ lookup('file','./templates/tcp/tcp_data_parsing.json') }}" - body_format: json - register: ts_pipeline_response - - - name: Ensures {{ remote_scripts_dir }} exists - become: yes - file: - path: "{{ remote_scripts_dir }}" - state: directory - - - name: Ensure {{ remote_scripts_dir }} directories are 777 - become: yes - shell: "chmod -R 777 transparent-security" - args: - chdir: "/etc" - - - name: Copy file sdn_attack_webhook.json to {{ remote_scripts_dir }} for updates - copy: - src: "{{ trans_sec_dir }}/playbooks/hcp/templates/sdn_attack_webhook.json" - dest: "{{ remote_scripts_dir }}/sdn_attack_webhook.json" - remote_src: yes - force: yes - - - name: Replace SDN Attack Webhook definition file for SDN port - become: yes - tags: sdnWebhookUpdateForPort - lineinfile: - path: "{{ remote_scripts_dir }}/sdn_attack_webhook.json" - regex: 'sdn_port' - line: '"port": "{{sdn_port}}",' - state: present - backup: yes - register: sdnWebhookUpdateForPort - - - name: Replace SDN Attack Webhook definition file for SDN IP - become: yes - tags: sdnWebhookUpdateForIP - lineinfile: - path: "{{ remote_scripts_dir }}/sdn_attack_webhook.json" - regex: 'sdn_ip' - line: '"host": "{{sdn_ip}}"' - state: present - backup: yes - register: sdnWebhookUpdateForIP - - - name: Slurp updated sdn webhook definition and save it to be used in POST request. - slurp: - src: "{{ remote_scripts_dir }}/sdn_attack_webhook.json" - register: slurped_updated_sdn_webhook_def - - - name: Set updated_sdn_webhook_def as fact - set_fact: - updated_sdn_webhook_def_contents: "{{ slurped_updated_sdn_webhook_def.content | b64decode }}" - - - name: Show SDN Webhook definition - debug: - var: updated_sdn_webhook_def_contents - - - - name: POST call to create SDN Aggregate attack webhook to attach to monitor for incoming data in Elasticsearch - uri: - url: "http://localhost:9200/_opendistro/_alerting/destinations" - method: POST - return_content: yes - status_code: 201 - body: "{{ updated_sdn_webhook_def_contents }}" - body_format: json - register: sdn_webhook_response - - - name: Set SDN Webhook call response - set_fact: - sdn_webhook_response: "{{ sdn_webhook_response.json }}" - - - name: Set SDN Webhook ID from response - set_fact: - sdn_webhook_id: "{{ sdn_webhook_response._id }}" - - - name: Fetch created SDN webhook ID for creating associated monitor - debug: - msg: " SDN Webhook response Id : {{ sdn_webhook_id }}" - - - name: Ensures {{ remote_scripts_dir }} exists - become: yes - file: - path: "{{ remote_scripts_dir }}" - state: directory - - - name: Ensure {{ remote_scripts_dir }} directories are 777 - become: yes - shell: "chmod -R 777 transparent-security" - args: - chdir: "/etc" - - - name: Copy file udp_monitor_def.json to {{ remote_scripts_dir }} for updates - copy: - src: "{{ trans_sec_dir }}/playbooks/hcp/templates/tcp/tcp_monitor_def.json" - dest: "{{ remote_scripts_dir }}/tcp_monitor_def.json" - remote_src: yes - force: yes - - - name: Replace SDN Webhook ID in TCP monitor definition file - become: yes - tags: monitorUpdate - lineinfile: - path: "{{ remote_scripts_dir }}/tcp_monitor_def.json" - regex: 'sdnWebhookResponseId' - line: ' "destination_id": "{{sdn_webhook_id}}",' - state: present - backup: yes - register: monitorUpdateOut - - - name: Slurp updated monitor definition and save it to be used in POST request. - slurp: - src: "{{ remote_scripts_dir }}/tcp_monitor_def.json" - register: slurped_updated_tcp_monitor_def - - - name: Set updated_tcp_monitor_def as fact - set_fact: - updated_tcp_monitor_def_contents: "{{ slurped_updated_tcp_monitor_def.content | b64decode }}" - - - name: Show Updated monitor definition - debug: - var: updated_tcp_monitor_def_contents - - - name: POST call to create Monitor,trigger and action for TCP DDOS Detection and Mitigation Pipeline for incoming data in Elasticsearch - uri: - url: "http://localhost:9200/_opendistro/_alerting/monitors" - method: POST - return_content: yes - status_code: 201 - body: "{{ updated_tcp_monitor_def_contents }}" - body_format: json - register: tcp_monitor_response diff --git a/playbooks/hcp/setup_ae_udp_and_tcp_pipeline.yml b/playbooks/hcp/setup_ae_udp_and_tcp_pipeline.yml deleted file mode 100644 index 3d5bf12a..00000000 --- a/playbooks/hcp/setup_ae_udp_and_tcp_pipeline.yml +++ /dev/null @@ -1,195 +0,0 @@ -# Copyright (c) 2020 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#Import playbook -- import_playbook: conf_ae_elk.yml - -- hosts: ae - gather_facts: no - tasks: - - name: POST call to create data mapping template for incoming UDP and TCP data in Elasticsearch - uri: - url: "http://localhost:9200/_template/packets" - method: PUT - return_content: yes - body: "{{ lookup('file','./templates/udp_and_tcp_data_mapping.json') }}" - body_format: json - register: udp_template_response - - - name: POST call to create script function to generate drop report hash - uri: - url: "http://localhost:9200/_scripts/calculateKeyHash" - method: POST - return_content: yes - body: "{{ lookup('file','./templates/drop_report_hash_script.json') }}" - body_format: json - register: drop_report_hash_response - - - name: POST call to create data parsing pipeline for UDP and TCP v4 and UDP v6 for incoming data in Elasticsearch - uri: - url: "http://localhost:9200/_ingest/pipeline/ts_parsing" - method: PUT - return_content: yes - body: "{{ lookup('file','./templates/udp_and_tcp_data_parsing.json') }}" - body_format: json - register: ts_pipeline_response - - - name: Ensures {{ remote_scripts_dir }} exists - become: yes - file: - path: "{{ remote_scripts_dir }}" - state: directory - - - name: Ensure {{ remote_scripts_dir }} directories are 777 - become: yes - shell: "chmod -R 777 transparent-security" - args: - chdir: "/etc" - - - name: Copy file sdn_attack_webhook.json to {{ remote_scripts_dir }} for updates - copy: - src: "{{ trans_sec_dir }}/playbooks/ae/templates/sdn_attack_webhook.json" - dest: "{{ remote_scripts_dir }}/sdn_attack_webhook.json" - remote_src: yes - force: yes - - - name: Replace SDN Attack Webhook definition file for SDN port - become: yes - tags: sdnWebhookUpdateForPort - lineinfile: - path: "{{ remote_scripts_dir }}/sdn_attack_webhook.json" - regex: 'sdn_port' - line: '"port": "{{sdn_port}}",' - state: present - backup: yes - register: sdnWebhookUpdateForPort - - - name: Replace SDN Attack Webhook definition file for SDN IP - become: yes - tags: sdnWebhookUpdateForIP - lineinfile: - path: "{{ remote_scripts_dir }}/sdn_attack_webhook.json" - regex: 'sdn_ip' - line: '"host": "{{sdn_ip}}"' - state: present - backup: yes - register: sdnWebhookUpdateForIP - - - name: Slurp updated sdn webhook definition and save it to be used in POST request. - slurp: - src: "{{ remote_scripts_dir }}/sdn_attack_webhook.json" - register: slurped_updated_sdn_webhook_def - - - name: Set updated_sdn_webhook_def as fact - set_fact: - updated_sdn_webhook_def_contents: "{{ slurped_updated_sdn_webhook_def.content | b64decode }}" - - - name: Show SDN Webhook definition - debug: - var: updated_sdn_webhook_def_contents - - - name: POST call to create SDN Aggregate attack webhook to attach to monitor for incoming data in Elasticsearch - uri: - url: "http://localhost:9200/_opendistro/_alerting/destinations" - method: POST - return_content: yes - status_code: 201 - body: "{{ updated_sdn_webhook_def_contents }}" - body_format: json - register: sdn_webhook_response - - - name: Set SDN Webhook call response - set_fact: - sdn_webhook_response: "{{ sdn_webhook_response.json }}" - - - name: Set SDN Webhook ID from response - set_fact: - sdn_webhook_id: "{{ sdn_webhook_response._id }}" - - - name: Fetch created SDN webhook ID for creating associated monitor - debug: - msg: " SDN Webhook response Id : {{ sdn_webhook_id }}" - - - - name: Copy file udp_monitor_def.json to {{ remote_scripts_dir }} for updates - copy: - src: "{{ trans_sec_dir }}/playbooks/ae/templates/udp_and_tcp_monitor_def.json" - dest: "{{ remote_scripts_dir }}/udp_and_tcp_monitor_def.json" - remote_src: yes - force: yes - - - name: Replace SDN Webhook ID in UDP monitor definition file for TCPv4 trigger - become: yes - tags: monitorUpdate - lineinfile: - path: "{{ remote_scripts_dir }}/udp_and_tcp_monitor_def.json" - regex: 'sdnWebhookResponseIdTCPv4' - line: ' "destination_id": "{{sdn_webhook_id}}",' - state: present - backup: yes - register: monitorUpdateOut - - - name: Replace SDN Webhook ID in UDP monitor definition file for TCPv6 trigger - become: yes - tags: monitorUpdate - lineinfile: - path: "{{ remote_scripts_dir }}/udp_and_tcp_monitor_def.json" - regex: 'sdnWebhookResponseIdTCPv6' - line: ' "destination_id": "{{sdn_webhook_id}}",' - state: present - backup: yes - register: monitorUpdateOut - - - name: Replace SDN Webhook ID in UDP monitor definition file for UDPv4 trigger - become: yes - tags: monitorUpdate - lineinfile: - path: "{{ remote_scripts_dir }}/udp_and_tcp_monitor_def.json" - regex: 'sdnWebhookResponseIdUDPv4' - line: ' "destination_id": "{{sdn_webhook_id}}",' - state: present - backup: yes - register: monitorUpdateOut - - - name: Replace SDN Webhook ID in UDP monitor definition file for UDPv6 trigger - become: yes - tags: monitorUpdate - lineinfile: - path: "{{ remote_scripts_dir }}/udp_and_tcp_monitor_def.json" - regex: 'sdnWebhookResponseIdUDPv6' - line: ' "destination_id": "{{sdn_webhook_id}}",' - state: present - backup: yes - register: monitorUpdateOut - - - name: Slurp updated monitor definition and save it to be used in POST request. - slurp: - src: "{{ remote_scripts_dir }}/udp_and_tcp_monitor_def.json" - register: slurped_updated_udp_and_tcp_monitor_def - - - name: Set updated_udp_monitor_def as fact - set_fact: - updated_udp_and_tcp_monitor_def_contents: "{{ slurped_updated_udp_and_tcp_monitor_def.content | b64decode }}" - - - name: Show Updated monitor definition for UDP and TCP pipeline - debug: - var: updated_udp_and_tcp_monitor_def_contents - - - name: POST call to create Monitor,trigger and action for UDP DDOS Detection and Mitigation Pipeline for incoming data in Elasticsearch - uri: - url: "http://localhost:9200/_opendistro/_alerting/monitors" - method: POST - return_content: yes - status_code: 201 - body: "{{ updated_udp_and_tcp_monitor_def_contents }}" - body_format: json - register: udp_and_tcp_monitor_response diff --git a/playbooks/hcp/setup_ae_udp_pipeline.yml b/playbooks/hcp/setup_ae_udp_pipeline.yml deleted file mode 100644 index e194daed..00000000 --- a/playbooks/hcp/setup_ae_udp_pipeline.yml +++ /dev/null @@ -1,162 +0,0 @@ -# Copyright (c) 2020 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#Import playbook -- import_playbook: conf_ae_elk.yml - -- hosts: ae - gather_facts: no - tasks: - - name: POST call to create data mapping template for incoming data in Elasticsearch - uri: - url: "http://localhost:9200/_template/packets" - method: PUT - return_content: yes - body: "{{ lookup('file','./templates/udp/udp_data_mapping.json') }}" - body_format: json - register: udp_template_response - - - name: POST call to create script function to generate drop report hash - uri: - url: "http://localhost:9200/_scripts/calculateKeyHash" - method: POST - return_content: yes - body: "{{ lookup('file','./templates/drop_report_hash_script.json') }}" - body_format: json - register: drop_report_hash_response - - - name: POST call to create data parsing pipeline for UDP v4 and UDP v6 for incoming data in Elasticsearch - uri: - url: "http://localhost:9200/_ingest/pipeline/ts_parsing" - method: PUT - return_content: yes - body: "{{ lookup('file','./templates/udp/udp_data_parsing.json') }}" - body_format: json - register: ts_pipeline_response - - - name: Ensures {{ remote_scripts_dir }} exists - become: yes - file: - path: "{{ remote_scripts_dir }}" - state: directory - - - name: Ensure {{ remote_scripts_dir }} directories are 777 - become: yes - shell: "chmod -R 777 transparent-security" - args: - chdir: "/etc" - - - name: Copy file sdn_attack_webhook.json to {{ remote_scripts_dir }} for updates - copy: - src: "{{ trans_sec_dir }}/playbooks/hcp/templates/sdn_attack_webhook.json" - dest: "{{ remote_scripts_dir }}/sdn_attack_webhook.json" - remote_src: yes - force: yes - - - name: Replace SDN Attack Webhook definition file for SDN port - become: yes - tags: sdnWebhookUpdateForPort - lineinfile: - path: "{{ remote_scripts_dir }}/sdn_attack_webhook.json" - regex: 'sdn_port' - line: '"port": "{{sdn_port}}",' - state: present - backup: yes - register: sdnWebhookUpdateForPort - - - name: Replace SDN Attack Webhook definition file for SDN IP - become: yes - tags: sdnWebhookUpdateForIP - lineinfile: - path: "{{ remote_scripts_dir }}/sdn_attack_webhook.json" - regex: 'sdn_ip' - line: '"host": "{{sdn_ip}}"' - state: present - backup: yes - register: sdnWebhookUpdateForIP - - - name: Slurp updated sdn webhook definition and save it to be used in POST request. - slurp: - src: "{{ remote_scripts_dir }}/sdn_attack_webhook.json" - register: slurped_updated_sdn_webhook_def - - - name: Set updated_sdn_webhook_def as fact - set_fact: - updated_sdn_webhook_def_contents: "{{ slurped_updated_sdn_webhook_def.content | b64decode }}" - - - name: Show SDN Webhook definition - debug: - var: updated_sdn_webhook_def_contents - - - name: POST call to create SDN Aggregate attack webhook to attach to monitor for incoming data in Elasticsearch - uri: - url: "http://localhost:9200/_opendistro/_alerting/destinations" - method: POST - return_content: yes - status_code: 201 - body: "{{ updated_sdn_webhook_def_contents }}" - body_format: json - register: sdn_webhook_response - - - name: Set SDN Webhook call response - set_fact: - sdn_webhook_response: "{{ sdn_webhook_response.json }}" - - - name: Set SDN Webhook ID from response - set_fact: - sdn_webhook_id: "{{ sdn_webhook_response._id }}" - - - name: Fetch created SDN webhook ID for creating associated monitor - debug: - msg: " SDN Webhook response Id : {{ sdn_webhook_id }}" - - - - name: Copy file udp_monitor_def.json to {{ remote_scripts_dir }} for updates - copy: - src: "{{ trans_sec_dir }}/playbooks/hcp/templates/udp/udp_monitor_def.json" - dest: "{{ remote_scripts_dir }}/udp_monitor_def.json" - remote_src: yes - force: yes - - - name: Replace SDN Webhook ID in UDP monitor definition file - become: yes - tags: monitorUpdate - lineinfile: - path: "{{ remote_scripts_dir }}/udp_monitor_def.json" - regex: 'sdnWebhookResponseId' - line: ' "destination_id": "{{sdn_webhook_id}}",' - state: present - backup: yes - register: monitorUpdateOut - - - name: Slurp updated monitor definition and save it to be used in POST request. - slurp: - src: "{{ remote_scripts_dir }}/udp_monitor_def.json" - register: slurped_updated_udp_monitor_def - - - name: Set updated_udp_monitor_def as fact - set_fact: - updated_udp_monitor_def_contents: "{{ slurped_updated_udp_monitor_def.content | b64decode }}" - - - name: Show Updated monitor definition - debug: - var: updated_udp_monitor_def_contents - - - name: POST call to create Monitor,trigger and action for UDP DDOS Detection and Mitigation Pipeline for incoming data in Elasticsearch - uri: - url: "http://localhost:9200/_opendistro/_alerting/monitors" - method: POST - return_content: yes - status_code: 201 - body: "{{ updated_udp_monitor_def_contents }}" - body_format: json - register: udp_monitor_response diff --git a/playbooks/hcp/start_ae_elk.yml b/playbooks/hcp/start_ae_elk.yml deleted file mode 100644 index 0461bee5..00000000 --- a/playbooks/hcp/start_ae_elk.yml +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright (c) 2020 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -- hosts: ae - gather_facts: no - vars: - kibana_wait: "{{ wait_for_kibana | default(180) }}" - tasks: - - name: "Start Elasticsearch service" - become: yes - service: - name: elasticsearch - state: started - enabled: yes - - - name: "Start Kibana service" - become: yes - service: - name: kibana - state: started - enabled: yes - - - name: Wait for Kibana to startup at port 5601 - wait_for: - port: 5601 - timeout: 600 - - - name: Wait for kibana - pause: - seconds: "{{ kibana_wait }}" diff --git a/playbooks/hcp/templates/drop_report_hash_script.json b/playbooks/hcp/templates/drop_report_hash_script.json deleted file mode 100644 index 457b5bfd..00000000 --- a/playbooks/hcp/templates/drop_report_hash_script.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "script" : { - "lang" : "painless", - "source" : """ -int[] HS() { - return new int[] { - (int) 0x6a09e667L, - (int) 0xbb67ae85L, - (int) 0x3c6ef372L, - (int) 0xa54ff53aL, - (int) 0x510e527fL, - (int) 0x9b05688cL, - (int) 0x1f83d9abL, - (int) 0x5be0cd19L - }; -} -int[] KS() { -return new int[] { - (int) 0x428a2f98L, (int) 0x71374491L, (int) 0xb5c0fbcfL, (int) 0xe9b5dba5L, - (int) 0x3956c25bL, (int) 0x59f111f1L, (int) 0x923f82a4L, (int) 0xab1c5ed5L, - (int) 0xd807aa98L, (int) 0x12835b01L, (int) 0x243185beL, (int) 0x550c7dc3L, - (int) 0x72be5d74L, (int) 0x80deb1feL, (int) 0x9bdc06a7L, (int) 0xc19bf174L, - (int) 0xe49b69c1L, (int) 0xefbe4786L, (int) 0x0fc19dc6L, (int) 0x240ca1ccL, - (int) 0x2de92c6fL, (int) 0x4a7484aaL, (int) 0x5cb0a9dcL, (int) 0x76f988daL, - (int) 0x983e5152L, (int) 0xa831c66dL, (int) 0xb00327c8L, (int) 0xbf597fc7L, - (int) 0xc6e00bf3L, (int) 0xd5a79147L, (int) 0x06ca6351L, (int) 0x14292967L, - (int) 0x27b70a85L, (int) 0x2e1b2138L, (int) 0x4d2c6dfcL, (int) 0x53380d13L, - (int) 0x650a7354L, (int) 0x766a0abbL, (int) 0x81c2c92eL, (int) 0x92722c85L, - (int) 0xa2bfe8a1L, (int) 0xa81a664bL, (int) 0xc24b8b70L, (int) 0xc76c51a3L, - (int) 0xd192e819L, (int) 0xd6990624L, (int) 0xf40e3585L, (int) 0x106aa070L, - (int) 0x19a4c116L, (int) 0x1e376c08L, (int) 0x2748774cL, (int) 0x34b0bcb5L, - (int) 0x391c0cb3L, (int) 0x4ed8aa4aL, (int) 0x5b9cca4fL, (int) 0x682e6ff3L, - (int) 0x748f82eeL, (int) 0x78a5636fL, (int) 0x84c87814L, (int) 0x8cc70208L, - (int) 0x90befffaL, (int) 0xa4506cebL, (int) 0xbef9a3f7L, (int) 0xc67178f2L - }; -} -byte[] digest(byte[] data, byte[] block, int[] words) { - byte[] padded = padMessage(data); - int[] hs = HS(); - - for (int i = 0; i < padded.length / 64; ++i) { - int[] registers = new int[8]; - for (int k = 0; k < 8; ++k) - registers[k] = hs[k]; - - System.arraycopy(padded, 64 * i, block, 0, 64); - - setupWords(block, words); - int[] ks = KS(); - - for (int j = 0; j < 64; ++j) { - iterate(registers, words, j, ks); - } - - for (int j = 0; j < 8; ++j) { - hs[j] += registers[j]; - } - } - byte[] hash = new byte[32]; - for (int i = 0; i < 8; i++) { - System.arraycopy(intToBytes(hs[i]), 0, hash, 4 * i, 4); - } - return hash; -} -void setupWords(byte[] block, int[] words) { - for (int j = 0; j < 16; j++) { - words[j] = 0; - for (int m = 0; m < 4; m++) { - words[j] |= ((block[j * 4 + m] & 255) << (24 - m * 8)); - } - } - for (int j = 16; j < 64; ++j) { - int s0 = Integer.rotateRight(words[j - 15], 7) ^ - Integer.rotateRight(words[j - 15], 18) ^ - (words[j - 15] >>> 3); - int s1 = Integer.rotateRight(words[j - 2], 17) ^ - Integer.rotateRight(words[j - 2], 19) ^ - (words[j - 2] >>> 10); - words[j] = words[j - 16] + s0 + words[j - 7] + s1; - } -} -void iterate(int[] registers, int[] words, int j, int[] ks) { - int S0 = Integer.rotateRight(registers[0], 2) ^ - Integer.rotateRight(registers[0], 13) ^ - Integer.rotateRight(registers[0], 22); - int maj = (registers[0] & registers[1]) ^ (registers[0] & registers[2]) ^ (registers[1] & registers[2]); - int temp2 = S0 + maj; - int S1 = Integer.rotateRight(registers[4], 6) ^ - Integer.rotateRight(registers[4], 11) ^ - Integer.rotateRight(registers[4], 25); - int ch = (registers[4] & registers[5]) ^ (~registers[4] & registers[6]); - int temp1 = registers[7] + S1 + ch + ks[j] + words[j]; - registers[7] = registers[6]; - registers[6] = registers[5]; - registers[5] = registers[4]; - registers[4] = registers[3] + temp1; - registers[3] = registers[2]; - registers[2] = registers[1]; - registers[1] = registers[0]; - registers[0] = temp1 + temp2; -} -byte[] padMessage(byte[] data) { - int length = data.length; - int tail = length % 64; - int padding; - if ((64 - tail >= 9)) { - padding = 64 - tail; - } else { - padding = 128 - tail; - } - byte[] pad = new byte[padding]; - pad[0] = (byte) 128; - long bits = length * 8; - for (int i = 0; i < 8; i++) { - pad[pad.length - 1 - i] = (byte) ((bits >>> (8 * i)) & 255); - } - byte[] output = new byte[length + padding]; - System.arraycopy(data, 0, output, 0, length); - System.arraycopy(pad, 0, output, length, pad.length); - return output; -} -byte[] intToBytes(int i) { - byte[] b = new byte[4]; - for (int c = 0; c < 4; c++) { - b[c] = (byte) ((i >>> (56 - 8 * c)) & 255); - } - return b; -} -String bytesToHex(byte[] bytes) { - char[] hexArray = "0123456789abcdef".toCharArray(); - char[] hexChars = new char[bytes.length * 2]; - for ( int j = 0; j < bytes.length; j++ ) { - int v = bytes[j] & 255; - hexChars[j * 2] = hexArray[v >>> 4]; - hexChars[j * 2 + 1] = hexArray[v & 15]; - } - return String.copyValueOf(hexChars); -} -byte[] charArrayToByteArray(char[] charArray) { -byte[] byteArray = new byte[charArray.length]; -for(int i= 0; i < charArray.length; i++) { - byteArray[i] = (byte)(255 & (int)charArray[i]); - } -return byteArray; -} -byte[] block = new byte[64]; -int[] words = new int[64]; -String dropHashInput = ctx.ts.dropHashInput; -if(dropHashInput != null && ctx.ts.isDropReport == false) -{ -byte[] data = charArrayToByteArray(dropHashInput.toCharArray()); -byte[] res = digest(data, block, words); -String sha256Str = bytesToHex(res); -ctx.ts.sha256Hash = sha256Str; -} - """ - } -} diff --git a/playbooks/hcp/templates/index_pattern_request_body.json b/playbooks/hcp/templates/index_pattern_request_body.json deleted file mode 100644 index 135f81a2..00000000 --- a/playbooks/hcp/templates/index_pattern_request_body.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "attributes": { - "title": "packets-*", - "timeFieldName": "timestamp" - } -} diff --git a/playbooks/hcp/templates/opendistro_elasticsearch.sh b/playbooks/hcp/templates/opendistro_elasticsearch.sh deleted file mode 100644 index f52c022a..00000000 --- a/playbooks/hcp/templates/opendistro_elasticsearch.sh +++ /dev/null @@ -1,113 +0,0 @@ -# ======================== Elasticsearch Configuration ========================= -# -# NOTE: Elasticsearch comes with reasonable defaults for most settings. -# Before you set out to tweak and tune the configuration, make sure you -# understand what are you trying to accomplish and the consequences. -# -# The primary way of configuring a node is via this file. This template lists -# the most important settings you may want to configure for a production cluster. -# -# Please consult the documentation for further information on configuration options: -# https://www.elastic.co/guide/en/elasticsearch/reference/index.html -# -# ---------------------------------- Cluster ----------------------------------- -# -# Use a descriptive name for your cluster: -# -cluster.name: ae-elasticsearch -# -# ------------------------------------ Node ------------------------------------ -# -# Use a descriptive name for the node: -# -#node.name: node-1 -# -# Add custom attributes to the node: -# -#node.attr.rack: r1 -# -# ----------------------------------- Paths ------------------------------------ -# -# Path to directory where to store the data (separate multiple locations by comma): -# -path.data: /var/lib/elasticsearch -# -# Path to log files: -# -path.logs: /var/log/elasticsearch -# -# ----------------------------------- Memory ----------------------------------- -# -# Lock the memory on startup: -# -#bootstrap.memory_lock: true -# -# Make sure that the heap size is set to about half the memory available -# on the system and that the owner of the process is allowed to use this -# limit. -# -# Elasticsearch performs poorly when the system is swapping the memory. -# -# ---------------------------------- Network ----------------------------------- -# -# Set the bind address to a specific IP (IPv4 or IPv6): -# -#network.host: 0.0.0.0 -# -# Set a custom port for HTTP: -# -#http.port: 9200 -# -# For more information, consult the network module documentation. -# -# --------------------------------- Discovery ---------------------------------- -# -# Pass an initial list of hosts to perform discovery when this node is started: -# The default list of hosts is ["127.0.0.1", "[::1]"] -# -#discovery.seed_hosts: ["127.0.0.1"] -# -# Bootstrap the cluster using an initial set of master-eligible nodes: -# -#cluster.initial_master_nodes: ["node-1", "node-2"] -# -# For more information, consult the discovery and cluster formation module documentation. -# -# ---------------------------------- Gateway ----------------------------------- -# -# Block initial recovery after a full cluster restart until N nodes are started: -# -#gateway.recover_after_nodes: 3 -# -# For more information, consult the gateway module documentation. -# -# ---------------------------------- Various ----------------------------------- -# -# Require explicit names when deleting indices: -# -#action.destructive_requires_name: true -######## Start OpenDistro for Elasticsearch Security Demo Configuration ######## -# WARNING: revise all the lines below before you go into production -#opendistro_security.disabled: true -#opendistro_security.ssl.transport.pemcert_filepath: esnode.pem -#opendistro_security.ssl.transport.pemkey_filepath: esnode-key.pem -#opendistro_security.ssl.transport.pemtrustedcas_filepath: root-ca.pem -#opendistro_security.ssl.transport.enforce_hostname_verification: false -#opendistro_security.ssl.http.enabled: true -#opendistro_security.ssl.http.pemcert_filepath: esnode.pem -#opendistro_security.ssl.http.pemkey_filepath: esnode-key.pem -#opendistro_security.ssl.http.pemtrustedcas_filepath: root-ca.pem -#opendistro_security.allow_unsafe_democertificates: true -#opendistro_security.allow_default_init_securityindex: true -#opendistro_security.authcz.admin_dn: -# - CN=kirk,OU=client,O=client,L=test, C=de -#opendistro_security.audit.type: internal_elasticsearch -#opendistro_security.enable_snapshot_restore_privilege: true -#opendistro_security.check_snapshot_restore_write_privileges: true -#opendistro_security.restapi.roles_enabled: ["all_access", "security_rest_api_access"] -#opendistro_security.system_indices.enabled: true -#opendistro_security.system_indices.indices: [".opendistro-alerting-config", ".opendistro-alerting-alert*"] -cluster.routing.allocation.disk.threshold_enabled: false -node.max_local_storage_nodes: 3 -######## End OpenDistro for Elasticsearch Security Demo Configuration ######## - diff --git a/playbooks/hcp/templates/opendistro_kibana.sh b/playbooks/hcp/templates/opendistro_kibana.sh deleted file mode 100644 index 1052065f..00000000 --- a/playbooks/hcp/templates/opendistro_kibana.sh +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). -# You may not use this file except in compliance with the License. -# A copy of the License is located at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# or in the "license" file accompanying this file. This file is distributed -# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either -# express or implied. See the License for the specific language governing -# permissions and limitations under the License. -# Description: -# Default Kibana configuration for Open Distro. - -server.port: 5601 -server.host: 0.0.0.0 -elasticsearch.hosts: http://localhost:9200 -elasticsearch.ssl.verificationMode: none -#elasticsearch.username: admin -#elasticsearch.password: admin -#elasticsearch.requestHeadersWhitelist: ["securitytenant","Authorization"] -#opendistro_security.multitenancy.enabled: true -#opendistro_security.multitenancy.tenants.preferred: ["Private", "Global"] -#opendistro_security.readonly_mode.roles: ["kibana_read_only"] -# Use this setting if you are running kibana without https -#opendistro_security.cookie.secure: false -newsfeed.enabled: false -telemetry.optIn: false -telemetry.enabled: false - diff --git a/playbooks/hcp/templates/query_monitor_id.json b/playbooks/hcp/templates/query_monitor_id.json deleted file mode 100644 index 1a9f2041..00000000 --- a/playbooks/hcp/templates/query_monitor_id.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "query": { - "match" : { - "monitor.name": "DDOS-UDP-TCP-Monitor" - } - } -} diff --git a/playbooks/hcp/templates/sdn_attack_webhook.json b/playbooks/hcp/templates/sdn_attack_webhook.json deleted file mode 100644 index 9fded291..00000000 --- a/playbooks/hcp/templates/sdn_attack_webhook.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "type": "custom_webhook", - "name": "SDN_Webhook", - "custom_webhook": { - "path": "aggAttack", - "header_params": { - "Content-Type": "application/json" - }, - "scheme": "HTTP", - "port": "sdn_port", - "host": "sdn_ip" - } -} diff --git a/playbooks/hcp/templates/tcp/tcp_data_mapping.json b/playbooks/hcp/templates/tcp/tcp_data_mapping.json deleted file mode 100644 index 80808a29..00000000 --- a/playbooks/hcp/templates/tcp/tcp_data_mapping.json +++ /dev/null @@ -1,292 +0,0 @@ -{ - "index_patterns" : [ - "packets-*" - ], - "settings" : { - "index" : { - "number_of_replicas" : "0", - "default_pipeline" : "ts_tcp_parsing", - "max_script_fields" : "200" - } - }, - "mappings" : { - "dynamic" : "false", - "properties" : { - "layers" : { - "properties" : { - "data_raw" : { - "type" : "keyword" - }, - "data" : { - "properties" : { - "data_data_data_raw" : { - "type" : "keyword" - }, - "data_data_len" : { - "type" : "keyword" - }, - "data_data_data" : { - "type" : "keyword" - } - } - } - } - }, - "timestamp" : { - "type" : "date" - }, - "ts" : { - "properties" : { - "INTMetadataSourceMetadataReserved" : { - "type" : "keyword" - }, - - "UDPIntShimHeaderType" : { - "type" : "keyword" - }, - "telemetryReportLength" : { - "type" : "keyword" - }, - "data" : { - "type" : "keyword" - }, - "ethernetHeaderDestMac" : { - "type" : "keyword" - }, - "telemetryRsvd" : { - "type" : "keyword" - }, - "UDGSrcPort" : { - "type" : "keyword" - }, - "INTMetadataSourceMetadataSwitchId" : { - "type" : "keyword" - }, - "ethernetHeaderSrcMac" : { - "type" : "keyword" - }, - "UDPIntShimHeaderNextProto" : { - "type" : "keyword" - }, - "isIpv4" : { - "type" : "keyword" - }, - "nextProto": { - "type" : "keyword" - }, - "isTCP" : { - "type" : "keyword" - }, - "isUDP" : { - "type" : "keyword" - }, - "TCPData" : { - "type" : "keyword" - }, - "TCPSrcPort" : { - "type" : "keyword" - }, - "TCPDestPort" : { - "type" : "keyword" - }, - "TCPIntShimHeaderNextProto" : { - "type" : "keyword" - }, - - "UDPIntMetaDataHeaderVersion" : { - "type" : "keyword" - }, - "IPvNextHdrProto" : { - "type" : "keyword" - }, - "telemetryReportVersion" : { - "type" : "keyword" - }, - "isIpv6" : { - "type" : "keyword" - }, - "IPv4SrcIP": { - "type" : "keyword" - }, - "IPv6SrcIP": { - "type" : "keyword" - }, - "IPv4DestIP":{ - "type" : "keyword" - }, - "IPv6DestIP":{ - "type" : "keyword" - }, - "INTMetadataSourceMetadataOriginatingMac" : { - "type" : "keyword" - }, - - "UDPIntMetaDataHeaderInstructionBitmap" : { - "type" : "keyword" - }, - "INTMetadataSourceMetadata" : { - "type" : "keyword" - }, - "IPvSrcAddr" : { - "type" : "keyword" - }, - "tsRawDataSize" : { - "type" : "keyword" - }, - "UDGPData" : { - "type" : "keyword" - }, - "UDGChecksum" : { - "type" : "keyword" - }, - "INTMetadataHeaderData" : { - "type" : "keyword" - }, - "IPvDestAddr" : { - "type" : "keyword" - }, - "telemetryInType" : { - "type" : "keyword" - }, - "telemetryReport" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderDSFlags" : { - "type" : "keyword" - }, - "telemetryDsMdStatus" : { - "type" : "keyword" - }, - "InBandNetworkTelemetryData" : { - "type" : "keyword" - }, - "UDPIntShimHeaderRes2" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderDSInstruction" : { - "type" : "keyword" - }, - "UDPIntShimHeaderRes1" : { - "type" : "keyword" - }, - "UDP2Data" : { - "type" : "keyword" - }, - "telemetryHwdId" : { - "type" : "keyword" - }, - - "telemetryVarOpt" : { - "type" : "keyword" - }, - "IPvData" : { - "type" : "keyword" - }, - "ethernetHeaderEtherType" : { - "type" : "keyword" - }, - "INTMetadataStackData" : { - "type" : "keyword" - }, - "IPvFlow" : { - "type" : "keyword" - }, - "UDPIntShimHeaderData" : { - "type" : "keyword" - }, - "INTMetadataStackSwitchID" : { - "type" : "keyword" - }, - "IPVersion" : { - "type" : "keyword" - }, - "IPvPayloadLength" : { - "type" : "keyword" - }, - "UDPIntShimHeaderLength" : { - "type" : "keyword" - }, - "UDPIntShimHeaderNPT" : { - "type" : "keyword" - }, - "UDGLength" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderM" : { - "type" : "keyword" - }, - - "telemetryD" : { - "type" : "keyword" - }, - "rawData" : { - "type" : "keyword" - }, - "telemetryF" : { - "type" : "keyword" - }, - "telemetryReportType" : { - "type" : "keyword" - }, - "telemetryI" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderPerhopMetadataLength" : { - "type" : "keyword" - }, - "telemetryQ" : { - "type" : "keyword" - }, - "telemetryDomainSpecificID" : { - "type" : "keyword" - }, - "udpLayersSize" : { - "type" : "keyword" - }, - "UDGDstPort" : { - "type" : "keyword" - }, - "IPvNextHopLimit" : { - "type" : "keyword" - }, - "telemetryDsMdBits" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderDomainSpecificID" : { - "type" : "keyword" - }, - "telemetryRepMdBits" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderRemainingHopCnt" : { - "type" : "keyword" - }, - "telemetrySequenceNumber" : { - "type" : "keyword" - }, - "telemetryMDLength" : { - "type" : "keyword" - }, - "IPvClass" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderD" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderE" : { - "type" : "keyword" - }, - "telemetryNodeId" : { - "type" : "keyword" - }, - "dataII" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderRes" : { - "type" : "keyword" - } - } - } - } - } -} diff --git a/playbooks/hcp/templates/tcp/tcp_data_parsing.json b/playbooks/hcp/templates/tcp/tcp_data_parsing.json deleted file mode 100644 index 4aad2c8f..00000000 --- a/playbooks/hcp/templates/tcp/tcp_data_parsing.json +++ /dev/null @@ -1,1332 +0,0 @@ -{ - "description" : "Transparent Security TCP v4 & v6 data parsing pipeline", - "processors" : [ - { - "set" : { - "field" : "ts.rawData", - "value" : "{{layers.data.data_data_data}}" - } - }, - { - "script" : { - "lang" : "painless", - "source" : "ctx.ts.telemetryReport =ctx.ts.rawData.substring(0, 48).replace(':','')", - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String hardwareIdStr = ctx.layers.data.data_data_data.replace(':','').substring(1, 3); - BigInteger hwdIdBigInt = new BigInteger(hardwareIdStr, 16); - BigInteger bitwiseAnd = new BigInteger('FC', 16); - BigInteger extractedHardwareId = hwdIdBigInt.and(bitwiseAnd); - String hwdIdStr = extractedHardwareId.toString(16); - ctx.ts.telemetryHwdId = hwdIdStr; - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String sequenceNumberPartStrI = tsRawData.substring(2, 3); - String sequenceNumberPartStrII = tsRawData.substring(3, 8); - String sequenceNumberPartStr = tsRawData.substring(2, 8); - BigInteger sequenceNumberBigInt = new BigInteger(sequenceNumberPartStr, 16); - BigInteger bitwiseAnd = new BigInteger('7FFFFFF', 16); - BigInteger updatedSequenceNumber = sequenceNumberBigInt.and(bitwiseAnd); - // String sequenceNumberHexStr = updatedSequenceNumber.toString(16); - ctx.ts.telemetrySequenceNumber = updatedSequenceNumber.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String nodeIdStr = tsRawData.substring(8, 16); - BigInteger nodeIdBigInt = new BigInteger(nodeIdStr, 16); - ctx.ts.telemetryNodeId =nodeIdBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String repTypeStr = tsRawData.substring(16, 17); - BigInteger repTypeBigInt = new BigInteger(repTypeStr, 16); - ctx.ts.telemetryReportType =repTypeBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String inTypeStr = tsRawData.substring(17, 18); - BigInteger inTypeBigInt = new BigInteger(inTypeStr, 16); - ctx.ts.telemetryInType = inTypeBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String reportLengthStr = tsRawData.substring(18, 20); - BigInteger reportLengthBigInt = new BigInteger(reportLengthStr, 16); - ctx.ts.telemetryReportLength = reportLengthBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String mdLengthStr = tsRawData.substring(20, 22); - BigInteger mdLengthBigInt = new BigInteger(mdLengthStr, 16); - ctx.ts.telemetryMDLength = mdLengthBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String dStr = tsRawData.substring(22, 23); - BigInteger dBigInt = new BigInteger(dStr, 16); - BigInteger bitwiseAnd = new BigInteger('8', 16); - BigInteger dRes = dBigInt.and(bitwiseAnd); - ctx.ts.telemetryD = dRes.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String qStr = tsRawData.substring(22, 23); - BigInteger qBigInt = new BigInteger(qStr, 16); - BigInteger bitwiseAnd = new BigInteger('4', 16); - BigInteger qRes = qBigInt.and(bitwiseAnd); - ctx.ts.telemetryQ = qRes.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String fStr = tsRawData.substring(22, 23); - BigInteger fBigInt = new BigInteger(fStr, 16); - BigInteger bitwiseAnd = new BigInteger('2', 16); - BigInteger fRes = fBigInt.and(bitwiseAnd); - ctx.ts.telemetryF = fRes.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String IStr = tsRawData.substring(22, 23); - BigInteger IBigInt = new BigInteger(IStr, 16); - BigInteger bitwiseAnd = new BigInteger('1', 16); - BigInteger IRes = IBigInt.and(bitwiseAnd); - ctx.ts.telemetryI = IRes.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String rsvdStr = tsRawData.substring(23, 24); - BigInteger rsvdBigInt = new BigInteger(rsvdStr, 16); - ctx.ts.telemetryRsvd = rsvdBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String repMdBitsStr = tsRawData.substring(24, 28); - BigInteger repMdBitsBigInt = new BigInteger(repMdBitsStr, 16); - ctx.ts.telemetryRepMdBits = repMdBitsBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String domainSpecificIDStr = tsRawData.substring(28, 32); - BigInteger domainSpecificIDBigInt = new BigInteger(domainSpecificIDStr, 16); - ctx.ts.telemetryDomainSpecificID = domainSpecificIDBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String dsMdBitsStr = tsRawData.substring(32, 36); - BigInteger dsMdBitsStrBigInt = new BigInteger(dsMdBitsStr, 16); - ctx.ts.telemetryDsMdBits = dsMdBitsStrBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String dsMdStatusStr = tsRawData.substring(36, 40); - BigInteger dsMdStatusBigInt = new BigInteger(dsMdStatusStr, 16); - ctx.ts.telemetryDsMdStatus = dsMdStatusBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String varOptStr = tsRawData.substring(40, 48); - BigInteger varOptBigInt = new BigInteger(varOptStr, 16); - ctx.ts.telemetryVarOpt = varOptBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.isIPv6 = true; - } else - { - ctx.ts.isIPv6 = false; - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.isIPv4 = true; - } else - { - ctx.ts.isIPv4 = false; - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String nextProtoStr = tsRawData.substring(138, 140); - BigInteger nextProtoBigInt = new BigInteger(nextProtoStr, 16); - ctx.ts.nextProto = nextProtoBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String nextProtoStr = tsRawData.substring(178, 180); - BigInteger nextProtoBigInt = new BigInteger(nextProtoStr, 16); - ctx.ts.nextProto = nextProtoBigInt.intValue(); - } - """, - "ignore_failure" : true - } - }, - - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - String tcpNextProtoStr = '6'; - int tcpNextProtoInt = Integer.parseInt(tcpNextProtoStr); - - if( etherType == '0800' && ipVersion == '4') - { - String nextProtoStr = tsRawData.substring(138, 140); - BigInteger nextProtoBigInt = new BigInteger(nextProtoStr, 16); - - if( ctx.ts.nextProto == tcpNextProtoInt ) - { - ctx.ts.isTCP = true; - }else { - ctx.ts.isTCP = false; - } - } - - if( etherType == '86dd' && ipVersion == '6') - { - String nextProtoStr = tsRawData.substring(178, 180); - BigInteger nextProtoBigInt = new BigInteger(nextProtoStr, 16); - String UDPNextProtoStr = ctx.ts.UDPIntShimHeaderNextProto.toString(); - if( ctx.ts.nextProto == tcpNextProtoInt ) - { - ctx.ts.isTCP = true; - }else { - ctx.ts.isTCP = false; - } - } - - """, - "ignore_failure" : true - } - }, - - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - String udpNextProtoStr = '17'; - int udpNextProtoInt = Integer.parseInt(udpNextProtoStr); - if( etherType == '0800' && ipVersion == '4') - { - String nextProtoStr = tsRawData.substring(138, 140); - BigInteger nextProtoBigInt = new BigInteger(nextProtoStr, 16); - String UDPNextProtoStr = ctx.ts.UDPIntShimHeaderNextProto.toString(); - if(ctx.ts.nextProto == udpNextProtoInt ) - { - ctx.ts.isUDP = true; - }else { - ctx.ts.isUDP = false; - } - } - - if( etherType == '86dd' && ipVersion == '6') - { - String nextProtoStr = tsRawData.substring(178, 180); - BigInteger nextProtoBigInt = new BigInteger(nextProtoStr, 16); - String UDPNextProtoStr = ctx.ts.UDPIntShimHeaderNextProto.toString(); - if(ctx.ts.nextProto == udpNextProtoInt ) - { - ctx.ts.isUDP = true; - }else { - ctx.ts.isUDP = false; - } - } - - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.data = tsRawData.substring(48, 252); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.data = tsRawData.substring(48, 292); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - ctx.ts.ethernetHeader = tsRawData.substring(48, 76); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - ctx.ts.ethernetHeaderDestMac = tsRawData.substring(48, 60); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - ctx.ts.ethernetHeaderSrcMac = tsRawData.substring(60, 72); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - ctx.ts.ethernetHeaderEtherType = tsRawData.substring(72, 76); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.IPvData = tsRawData.substring(76, 116); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.IPvData = tsRawData.substring(76, 156); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.IPVersion = tsRawData.substring(76, 77); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.IPVersion = tsRawData.substring(76, 77); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String IPv4SrcIPHexString = tsRawData.substring(100, 108); - String IPv4SrcIP = ""; - for(int i = 0; i < IPv4SrcIPHexString.length(); i = i + 2) { - IPv4SrcIP = IPv4SrcIP + Integer.valueOf(IPv4SrcIPHexString.substring(i, i+2), 16) + "."; - } - IPv4SrcIP = IPv4SrcIP.substring(0, IPv4SrcIP.length()-1); - ctx.ts.IPv4SrcIP = IPv4SrcIP; - } - if( etherType == '86dd' && ipVersion == '6') - { - String IPv6SrcIPHexString = tsRawData.substring(92,124); - String IPv6SrcIP = ""; - for(int i = 0; i < IPv6SrcIPHexString.length(); i = i + 2) { - IPv6SrcIP = IPv6SrcIP + Integer.valueOf(IPv6SrcIPHexString.substring(i, i+2), 16) + "."; - } - IPv6SrcIP = IPv6SrcIP.substring(0, IPv6SrcIP.length()-1); - ctx.ts.IPv6SrcIP = IPv6SrcIP; - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String IPv4DestIPHexString = tsRawData.substring(108, 116); - String IPv4DestIP = ""; - for(int i = 0; i < IPv4DestIPHexString.length(); i = i + 2) { - IPv4DestIP = IPv4DestIP + Integer.valueOf(IPv4DestIPHexString.substring(i, i+2), 16) + "."; - } - IPv4DestIP = IPv4DestIP.substring(0, IPv4DestIP.length()-1); - ctx.ts.IPv4DestIP = IPv4DestIP; - } - if( etherType == '86dd' && ipVersion == '6') - { - String IPv6DestIPHexString = tsRawData.substring(124,156); - String IPv6DestIP = ""; - for(int i = 0; i < IPv6DestIPHexString.length(); i = i + 2) { - IPv6DestIP = IPv6DestIP + Integer.valueOf(IPv6DestIPHexString.substring(i, i+2), 16) + "."; - } - IPv6DestIP = IPv6DestIP.substring(0, IPv6DestIP.length()-1); - ctx.ts.IPv6DestIP = IPv6DestIP; - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDGPData = tsRawData.substring(116, 132); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDGPData = tsRawData.substring(156, 172); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDGSrcPortStr = tsRawData.substring(116, 120); - BigInteger UDGSrcPortBigInt = new BigInteger(UDGSrcPortStr, 16); - ctx.ts.UDGSrcPort = UDGSrcPortBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDGSrcPortStr = tsRawData.substring(156, 160); - BigInteger UDGSrcPortBigInt = new BigInteger(UDGSrcPortStr, 16); - ctx.ts.UDGSrcPort = UDGSrcPortBigInt.intValue(); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDGDstPortStr = tsRawData.substring(120, 124); - BigInteger UDGDstPortBigInt = new BigInteger(UDGDstPortStr, 16); - ctx.ts.UDGDstPort = UDGDstPortBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDGDstPortStr = tsRawData.substring(160, 164); - BigInteger UDGDstPortBigInt = new BigInteger(UDGDstPortStr, 16); - ctx.ts.UDGDstPort = UDGDstPortBigInt.intValue(); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDGLengthStr = tsRawData.substring(124, 128); - BigInteger UDGLengthBigInt = new BigInteger(UDGLengthStr, 16); - ctx.ts.UDGLength = UDGLengthBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDGLengthStr = tsRawData.substring(164, 168); - BigInteger UDGLengthBigInt = new BigInteger(UDGLengthStr, 16); - ctx.ts.UDGLength = UDGLengthBigInt.intValue(); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDGChecksumStr = tsRawData.substring(128, 132); - BigInteger UDGChecksumBigInt = new BigInteger(UDGChecksumStr, 16); - ctx.ts.UDGChecksum = UDGChecksumBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDGChecksumStr = tsRawData.substring(168, 172); - BigInteger UDGChecksumBigInt = new BigInteger(UDGChecksumStr, 16); - ctx.ts.UDGChecksum = UDGChecksumBigInt.intValue(); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.InBandNetworkTelemetryData = tsRawData.substring(132, 196); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.InBandNetworkTelemetryData = tsRawData.substring(172, 236); - } - """, - "ignore_failure" : true - } - }, - - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDPIntShimHeaderData = tsRawData.substring(132, 140); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDPIntShimHeaderData = tsRawData.substring(172, 180); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDPIntShimHeaderType = tsRawData.substring(132, 133); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDPIntShimHeaderType = tsRawData.substring(172, 173); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String udpIntShimNPTStr = tsRawData.substring(133, 134); - BigInteger udpIntShimNPTBigInt = new BigInteger(udpIntShimNPTStr, 16); - BigInteger bitwiseAnd = new BigInteger('C', 16); - BigInteger result = udpIntShimNPTBigInt.and(bitwiseAnd); - ctx.ts.UDPIntShimHeaderNPT = result.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String udpIntShimNPTStr = tsRawData.substring(173, 174); - BigInteger udpIntShimNPTBigInt = new BigInteger(udpIntShimNPTStr, 16); - BigInteger bitwiseAnd = new BigInteger('C', 16); - BigInteger result = udpIntShimNPTBigInt.and(bitwiseAnd); - ctx.ts.UDPIntShimHeaderNPT = result.intValue(); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDPIntShimHeaderRes1 = tsRawData.substring(133, 134); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDPIntShimHeaderRes1 = tsRawData.substring(174, 175); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String lengthStr = tsRawData.substring(134, 136); - BigInteger lengthStrBigInt = new BigInteger(lengthStr, 16); - ctx.ts.UDPIntShimHeaderLength= lengthStrBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String lengthStr = tsRawData.substring(174, 176); - BigInteger lengthStrBigInt = new BigInteger(lengthStr, 16); - ctx.ts.UDPIntShimHeaderLength = lengthStrBigInt.intValue(); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDPIntShimHeaderRes2 = tsRawData.substring(136, 138); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDPIntShimHeaderRes2 = tsRawData.substring(176, 178); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String nextProtoStr = tsRawData.substring(138, 140); - BigInteger nextProtoBigInt = new BigInteger(nextProtoStr, 16); - ctx.ts.UDPIntShimHeaderNextProto = nextProtoBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String nextProtoStr = tsRawData.substring(178, 180); - BigInteger nextProtoBigInt = new BigInteger(nextProtoStr, 16); - ctx.ts.UDPIntShimHeaderNextProto = nextProtoBigInt.intValue(); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.INTMetadataHeaderData = tsRawData.substring(140, 164); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.INTMetadataHeaderData = tsRawData.substring(180, 204); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderVersionStr = tsRawData.substring(140, 141); - BigInteger UDPIntMetaDataHeaderVersionStrBigInt = new BigInteger(UDPIntMetaDataHeaderVersionStr, 16); - ctx.ts.UDPIntMetaDataHeaderVersion = UDPIntMetaDataHeaderVersionStrBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderVersionStr = tsRawData.substring(180, 181); - BigInteger UDPIntMetaDataHeaderVersionStrBigInt = new BigInteger(UDPIntMetaDataHeaderVersionStr, 16); - ctx.ts.UDPIntMetaDataHeaderVersion = UDPIntMetaDataHeaderVersionStrBigInt.intValue(); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderRes = tsRawData.substring(141, 142); - BigInteger UDPIntMetaDataHeaderResBigInt = new BigInteger(UDPIntMetaDataHeaderRes, 16); - BigInteger bitwiseAnd = new BigInteger('C', 16); - BigInteger result = UDPIntMetaDataHeaderResBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderRes = result.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderRes = tsRawData.substring(181, 182); - BigInteger UDPIntMetaDataHeaderResBigInt = new BigInteger(UDPIntMetaDataHeaderRes, 16); - BigInteger bitwiseAnd = new BigInteger('C', 16); - BigInteger result = UDPIntMetaDataHeaderResBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderRes = result.intValue(); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderD = tsRawData.substring(141, 142); - BigInteger UDPIntMetaDataHeaderDBigInt = new BigInteger(UDPIntMetaDataHeaderD, 16); - BigInteger bitwiseAnd = new BigInteger('2', 16); - BigInteger result = UDPIntMetaDataHeaderDBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderD = result.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderD = tsRawData.substring(181, 182); - BigInteger UDPIntMetaDataHeaderDBigInt = new BigInteger(UDPIntMetaDataHeaderD, 16); - BigInteger bitwiseAnd = new BigInteger('2', 16); - BigInteger result = UDPIntMetaDataHeaderDBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderD = result.intValue(); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderE = tsRawData.substring(141, 142); - BigInteger UDPIntMetaDataHeaderEBigInt = new BigInteger(UDPIntMetaDataHeaderE, 16); - BigInteger bitwiseAnd = new BigInteger('1', 16); - BigInteger result = UDPIntMetaDataHeaderEBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderE = result.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderE = tsRawData.substring(181, 182); - BigInteger UDPIntMetaDataHeaderEBigInt = new BigInteger(UDPIntMetaDataHeaderE, 16); - BigInteger bitwiseAnd = new BigInteger('1', 16); - BigInteger result = UDPIntMetaDataHeaderEBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderE = result.intValue(); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderM = tsRawData.substring(142, 143); - BigInteger UDPIntMetaDataHeaderMBigInt = new BigInteger(UDPIntMetaDataHeaderM, 16); - BigInteger bitwiseAnd = new BigInteger('8', 16); - BigInteger result = UDPIntMetaDataHeaderMBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderM = result.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderM = tsRawData.substring(182, 183); - BigInteger UDPIntMetaDataHeaderMBigInt = new BigInteger(UDPIntMetaDataHeaderM, 16); - BigInteger bitwiseAnd = new BigInteger('8', 16); - BigInteger result = UDPIntMetaDataHeaderMBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderM = result.intValue(); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderPerhopMetadataLengthStr = tsRawData.substring(144, 146); - BigInteger UDPIntMetaDataHeaderPerhopMetadataLengthBigInt = new BigInteger(UDPIntMetaDataHeaderPerhopMetadataLengthStr, 16); - BigInteger bitwiseAnd = new BigInteger('1F', 16); - BigInteger result = UDPIntMetaDataHeaderPerhopMetadataLengthBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderPerhopMetadataLength = result.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderPerhopMetadataLengthStr = tsRawData.substring(184, 186); - BigInteger UDPIntMetaDataHeaderPerhopMetadataLengthBigInt = new BigInteger(UDPIntMetaDataHeaderPerhopMetadataLengthStr, 16); - BigInteger bitwiseAnd = new BigInteger('1F', 16); - BigInteger result = UDPIntMetaDataHeaderPerhopMetadataLengthBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderPerhopMetadataLength = result.intValue(); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderRemainingHopCntStr = tsRawData.substring(146, 148); - BigInteger UDPIntMetaDataHeaderRemainingHopCntBigInt = new BigInteger(UDPIntMetaDataHeaderRemainingHopCntStr, 16); - ctx.ts.UDPIntMetaDataHeaderRemainingHopCnt = UDPIntMetaDataHeaderRemainingHopCntBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderRemainingHopCntStr = tsRawData.substring(186, 188); - BigInteger UDPIntMetaDataHeaderRemainingHopCntBigInt = new BigInteger(UDPIntMetaDataHeaderRemainingHopCntStr, 16); - ctx.ts.UDPIntMetaDataHeaderRemainingHopCnt = UDPIntMetaDataHeaderRemainingHopCntBigInt.intValue(); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderInstructionBitmapStr = tsRawData.substring(148, 152); - BigInteger UDPIntMetaDataHeaderInstructionBitmapBigInt = new BigInteger(UDPIntMetaDataHeaderInstructionBitmapStr, 16); - ctx.ts.UDPIntMetaDataHeaderInstructionBitmap = UDPIntMetaDataHeaderInstructionBitmapBigInt.toString(16); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderInstructionBitmapStr = tsRawData.substring(188, 192); - BigInteger UDPIntMetaDataHeaderInstructionBitmapBigInt = new BigInteger(UDPIntMetaDataHeaderInstructionBitmapStr, 16); - ctx.ts.UDPIntMetaDataHeaderInstructionBitmap = UDPIntMetaDataHeaderInstructionBitmapBigInt.toString(16); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderDomainSpecificIDStr = tsRawData.substring(152, 156); - BigInteger UDPIntMetaDataHeaderDomainSpecificIDBigInt = new BigInteger(UDPIntMetaDataHeaderDomainSpecificIDStr, 16); - ctx.ts.UDPIntMetaDataHeaderDomainSpecificID = UDPIntMetaDataHeaderDomainSpecificIDBigInt.toString(16); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderDomainSpecificIDStr = tsRawData.substring(192, 196); - BigInteger UDPIntMetaDataHeaderDomainSpecificIDBigInt = new BigInteger(UDPIntMetaDataHeaderDomainSpecificIDStr, 16); - ctx.ts.UDPIntMetaDataHeaderDomainSpecificID = UDPIntMetaDataHeaderDomainSpecificIDBigInt.toString(16); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderDSInstructionStr = tsRawData.substring(156, 160); - BigInteger UDPIntMetaDataHeaderDSInstructionBigInt = new BigInteger(UDPIntMetaDataHeaderDSInstructionStr, 16); - ctx.ts.UDPIntMetaDataHeaderDSInstruction = UDPIntMetaDataHeaderDSInstructionBigInt.toString(16); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderDSInstructionStr = tsRawData.substring(196, 200); - BigInteger UDPIntMetaDataHeaderDSInstructionBigInt = new BigInteger(UDPIntMetaDataHeaderDSInstructionStr, 16); - ctx.ts.UDPIntMetaDataHeaderDSInstruction = UDPIntMetaDataHeaderDSInstructionBigInt.toString(16); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderDSFlagsStr = tsRawData.substring(160, 164); - BigInteger UDPIntMetaDataHeaderDSFlagsBigInt = new BigInteger(UDPIntMetaDataHeaderDSFlagsStr, 16); - ctx.ts.UDPIntMetaDataHeaderDSFlags = UDPIntMetaDataHeaderDSFlagsBigInt.toString(16); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderDSFlagsStr = tsRawData.substring(200, 204); - BigInteger UDPIntMetaDataHeaderDSFlagsBigInt = new BigInteger(UDPIntMetaDataHeaderDSFlagsStr, 16); - ctx.ts.UDPIntMetaDataHeaderDSFlags = UDPIntMetaDataHeaderDSFlagsBigInt.toString(16); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.INTMetadataStackData = tsRawData.substring(164, 196); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.INTMetadataStackData = tsRawData.substring(204, 236); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.INTMetadataStackSwitchID = tsRawData.substring(164, 172); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.INTMetadataStackSwitchID = tsRawData.substring(204, 212); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.INTMetadataSourceMetadata = tsRawData.substring(172, 196); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.INTMetadataSourceMetadata = tsRawData.substring(212, 236); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.INTMetadataSourceMetadataSwitchId = tsRawData.substring(172, 180); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.INTMetadataSourceMetadataSwitchId = tsRawData.substring(212, 220); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String rawOriginatingMac = tsRawData.substring(180, 192); - StringBuilder formattedOriginatingMac = new StringBuilder(); - for(int i = 0; i < rawOriginatingMac.length();) - { - formattedOriginatingMac.append(rawOriginatingMac.substring(i++, ++i)); - if(i < rawOriginatingMac.length()) formattedOriginatingMac.append(":"); - } - ctx.ts.INTMetadataSourceMetadataOriginatingMac = formattedOriginatingMac.toString(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String rawOriginatingMac = tsRawData.substring(220, 232); - StringBuilder formattedOriginatingMac = new StringBuilder(); - for(int i = 0; i < rawOriginatingMac.length();) - { - formattedOriginatingMac.append(rawOriginatingMac.substring(i++, ++i)); - if(i < rawOriginatingMac.length()) formattedOriginatingMac.append(":"); - } - ctx.ts.INTMetadataSourceMetadataOriginatingMac = formattedOriginatingMac.toString(); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.INTMetadataSourceMetadataReserved = tsRawData.substring(192, 196); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.INTMetadataSourceMetadataReserved = tsRawData.substring(232, 236); - } - """, - "ignore_failure" : true - } - }, - - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - String tcpNextProtoStr = '6'; - int tcpNextProtoInt = Integer.parseInt(tcpNextProtoStr); - if( etherType == '0800' && ipVersion == '4' && ctx.ts.nextProto == tcpNextProtoInt ) - { - ctx.ts.TCPData = tsRawData.substring(196, 236); - } - if(etherType == '86dd' && ipVersion == '6' && ctx.ts.nextProto == tcpNextProtoInt ) - { - ctx.ts.TCPData = tsRawData.substring(236, 276); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - String tcpNextProtoStr = '6'; - int tcpNextProtoInt = Integer.parseInt(tcpNextProtoStr); - if( etherType == '0800' && ipVersion == '4' && ctx.ts.nextProto == tcpNextProtoInt ) - { - String TCPSrcPortStr = tsRawData.substring(196, 200); - BigInteger TCPSrcPortBigInt = new BigInteger(TCPSrcPortStr,16); - ctx.ts.TCPSrcPort = TCPSrcPortBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6' && ctx.ts.nextProto == tcpNextProtoInt ) - { - String TCPSrcPortStr = tsRawData.substring(236, 240); - BigInteger TCPSrcPortBigInt = new BigInteger(TCPSrcPortStr,16); - ctx.ts.TCPSrcPort = TCPSrcPortBigInt.intValue(); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - String tcpNextProtoStr = '6'; - int tcpNextProtoInt = Integer.parseInt(tcpNextProtoStr); - if( etherType == '0800' && ipVersion == '4' && ctx.ts.nextProto == tcpNextProtoInt ) - { - String TCPDstPortStr = tsRawData.substring(200, 204); - BigInteger TCPDstPortBigInt = new BigInteger(TCPDstPortStr,16); - ctx.ts.TCPDestPort = TCPDstPortBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6' && ctx.ts.nextProto == tcpNextProtoInt ) - { - String TCPDstPortStr = tsRawData.substring(240, 244); - BigInteger TCPDstPortBigInt = new BigInteger(TCPDstPortStr,16); - ctx.ts.TCPDestPort = TCPDstPortBigInt.intValue(); - } - """, - "ignore_failure" : true - } - }, - - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - String tcpNextProtoStr = '6'; - int tcpNextProtoInt = Integer.parseInt(tcpNextProtoStr); - if( etherType == '0800' && ipVersion == '4' && ctx.ts.nextProto == tcpNextProtoInt) - { - ctx.ts.dataII = tsRawData.substring(236, 288); - } - if(etherType == '86dd' && ipVersion == '6' && ctx.ts.nextProto == tcpNextProtoInt) - { - ctx.ts.dataII = tsRawData.substring(252, 304); - } - """, - "ignore_failure" : true - } - } - ] -} diff --git a/playbooks/hcp/templates/tcp/tcp_monitor_def.json b/playbooks/hcp/templates/tcp/tcp_monitor_def.json deleted file mode 100644 index 71061d74..00000000 --- a/playbooks/hcp/templates/tcp/tcp_monitor_def.json +++ /dev/null @@ -1,156 +0,0 @@ -{ - "type": "monitor", - "name": "DDOS-TCP-Monitor", - "enabled": true, - "schedule": { - "period": { - "interval": 1, - "unit": "MINUTES" - } - }, - "inputs": [{ - "search": { - "indices": ["packets-*"], - "query": { - "size": 0, - "query": { - "bool": { - "filter": [ - { - "range": { - "timestamp": { - "from": "{{period_end}}||-1m", - "to": "{{period_end}}", - "include_lower": true, - "include_upper": true, - "format": "epoch_millis", - "boost": 1 - } - } - }, - { - "wildcard": { - "ts.UDP2DstPort": { - "wildcard": "*5792*", - "boost": 1 - } - } - } - ], - "adjust_pure_negative": true, - "boost": 1 - } - }, - "_source": { - "includes": [ - "ts.TCPDestPort", - "ts.IPvDestAddr", - "ts.INTMetadataSourceMetadataOriginatingMac", - "ts.IPv4SrcIP", - "ts.IPv4DestIP", - "ts.IPv6SrcIP", - "ts.IPv6DestIP" - ], - "excludes": [] - }, - "aggregations": { - "TCPDestPortValueCount": { - "value_count": { - "field": "ts.TCPDestPort" - } - }, - "IPv4SrcIP": { - "terms": { - "field": "ts.IPv4SrcIP", - "size": 10, - "min_doc_count": 1, - "shard_min_doc_count": 0, - "show_term_doc_count_error": false, - "order": [ - { - "_count": "desc" - }, - { - "_key": "asc" - } - ] - } - }, - "IPv4DestIP": { - "terms": { - "field": "ts.IPv4DestIP", - "size": 10, - "min_doc_count": 1, - "shard_min_doc_count": 0, - "show_term_doc_count_error": false, - "order": [ - { - "_count": "desc" - }, - { - "_key": "asc" - } - ] - } - }, - "TCPDestPortValue": { - "terms": { - "field": "ts.TCPDestPort", - "size": 10, - "min_doc_count": 1, - "shard_min_doc_count": 0, - "show_term_doc_count_error": false, - "order": [ - { - "_count": "desc" - }, - { - "_key": "asc" - } - ] - } - }, - "OriginatingMacAddrTCPv4": { - "terms": { - "field": "ts.INTMetadataSourceMetadataOriginatingMac", - "size": 10, - "min_doc_count": 1, - "shard_min_doc_count": 0, - "show_term_doc_count_error": false, - "order": [ - { - "_count": "desc" - }, - { - "_key": "asc" - } - ] - } - } - } - } - } - }], - "triggers": [{ - "name": "DDOS-Trigger", - "severity": "1", - "condition": { - "script": { - "source": "ctx.results[0].aggregations.TCPDestPortValueCount.value >= 10", - "lang": "painless" - } - }, - "actions": [{ - "name": "Sdn-WebHook-Action", - "destination_id": "sdnWebhookResponseId", - "message_template": { - "source":"{\"event_action\":\"trigger\", \"dst_ip\":\"{{ctx.results.0.aggregations.IPv4DestIP.buckets.0.key}}\",\"dst_port\":\"{{ctx.results.0.aggregations.TCPDestPortValue.buckets.0.key}}\",\"src_ip\":\"{{ctx.results.0.aggregations.IPv4SrcIP.buckets.0.key}}\",\"src_mac\":\"{{ctx.results.0.aggregations.OriginatingMacAddrTCPv4.buckets.0.key}}\"}", - "lang": "mustache", - "options": { - "content_type": "application/json" - } - }, - "throttle_enabled": false - }] - }] -} diff --git a/playbooks/hcp/templates/udp/udp_data_mapping.json b/playbooks/hcp/templates/udp/udp_data_mapping.json deleted file mode 100644 index 24e756d9..00000000 --- a/playbooks/hcp/templates/udp/udp_data_mapping.json +++ /dev/null @@ -1,315 +0,0 @@ -{ - "order" : 0, - "index_patterns" : [ - "packets-*" - ], - "settings" : { - "index" : { - "number_of_replicas" : "0", - "default_pipeline" : "ts_parsing", - "max_script_fields" : "200" - } - }, - "mappings" : { - "dynamic" : "false", - "properties" : { - "layers" : { - "properties" : { - "data_raw" : { - "type" : "keyword" - }, - "data" : { - "properties" : { - "data_data_data_raw" : { - "type" : "keyword" - }, - "data_data_len" : { - "type" : "keyword" - }, - "data_data_data" : { - "type" : "keyword" - } - } - } - } - }, - "timestamp" : { - "type" : "date" - }, - "ts" : { - "properties" : { - "INTMetadataSourceMetadataReserved" : { - "type" : "keyword" - }, - "UDP2SrcPort" : { - "type" : "keyword" - }, - "UDPIntShimHeaderType" : { - "type" : "keyword" - }, - "telemetryReportLength" : { - "type" : "keyword" - }, - "data" : { - "type" : "keyword" - }, - "ethernetHeaderDestMac" : { - "type" : "keyword" - }, - "UDGSrcPort" : { - "type" : "keyword" - }, - "telemetryRsvd" : { - "type" : "keyword" - }, - "INTMetadataSourceMetadataSwitchId" : { - "type" : "keyword" - }, - "UDPIntShimHeaderNextProto" : { - "type" : "keyword" - }, - "IPv6SrcIP" : { - "type" : "keyword" - }, - "ethernetHeaderSrcMac" : { - "type" : "keyword" - }, - "isIPv4" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderVersion" : { - "type" : "keyword" - }, - "IPvNextHdrProto" : { - "type" : "keyword" - }, - "telemetryReportVersion" : { - "type" : "keyword" - }, - "isIPv6" : { - "type" : "keyword" - }, - "INTMetadataSourceMetadataOriginatingMac" : { - "type" : "keyword" - }, - "UDP2Checksum" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderInstructionBitmap" : { - "type" : "keyword" - }, - "INTMetadataSourceMetadata" : { - "type" : "keyword" - }, - "IPvSrcAddr" : { - "type" : "keyword" - }, - "UDGPData" : { - "type" : "keyword" - }, - "tsRawDataSize" : { - "type" : "keyword" - }, - "UDGChecksum" : { - "type" : "keyword" - }, - "IPvDestAddr" : { - "type" : "keyword" - }, - "INTMetadataHeaderData" : { - "type" : "keyword" - }, - "telemetryInType" : { - "type" : "keyword" - }, - "telemetryReportVarOptTimeStamp" : { - "type" : "keyword" - }, - "dropTelemetryReport" : { - "type" : "keyword" - }, - "telemetryReport" : { - "type" : "keyword" - }, - "telemetryDsMdStatus" : { - "type" : "keyword" - }, - "ts.isIPv4":{ - "type" : "keyword" - }, - "ts.isIPv6":{ - "type" : "keyword" - }, - "UDPIntMetaDataHeaderDSFlags" : { - "type" : "keyword" - }, - "InBandNetworkTelemetryData" : { - "type" : "keyword" - }, - "IPv4SrcIP" : { - "type" : "keyword" - }, - "UDPIntShimHeaderRes2" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderDSInstruction" : { - "type" : "keyword" - }, - "UDP2Data" : { - "type" : "keyword" - }, - "UDPIntShimHeaderRes1" : { - "type" : "keyword" - }, - "telemetryHwdId" : { - "type" : "keyword" - }, - "UDP2Length" : { - "type" : "keyword" - }, - "telemetryVarOpt" : { - "type" : "keyword" - }, - "ethernetHeaderEtherType" : { - "type" : "keyword" - }, - "IPvData" : { - "type" : "keyword" - }, - "INTMetadataStackData" : { - "type" : "keyword" - }, - "IPvFlow" : { - "type" : "keyword" - }, - "UDPIntShimHeaderData" : { - "type" : "keyword" - }, - "INTMetadataStackSwitchID" : { - "type" : "keyword" - }, - "IPVersion" : { - "type" : "keyword" - }, - - "IPvPayloadLength" : { - "type" : "keyword" - }, - "IPv4DestIP" : { - "type" : "keyword" - }, - "UDPIntShimHeaderLength" : { - "type" : "keyword" - }, - "UDPIntShimHeaderNPT" : { - "type" : "keyword" - }, - "UDGLength" : { - "type" : "keyword" - }, - "telemetryReportVarOptDropCount" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderM" : { - "type" : "keyword" - }, - "UDP2DstPort" : { - "type" : "keyword" - }, - "telemetryD" : { - "type" : "keyword" - }, - "dropReportHash" : { - "type" : "keyword" - }, - - "rawData" : { - "type" : "keyword" - }, - "telemetryF" : { - "type" : "keyword" - }, - "telemetryReportType" : { - "type" : "keyword" - }, - "telemetryI" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderPerhopMetadataLength" : { - "type" : "keyword" - }, - "telemetryQ" : { - "type" : "keyword" - }, - "telemetryDomainSpecificID" : { - "type" : "keyword" - }, - "udpLayersSize" : { - "type" : "keyword" - }, - "UDGDstPort" : { - "type" : "keyword" - }, - "IPvNextHopLimit" : { - "type" : "keyword" - }, - "dropHashInput" : { - "type" : "keyword" - }, - "telemetryDsMdBits" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderDomainSpecificID" : { - "type" : "keyword" - }, - "telemetryReportVarOpBsmd" : { - "type" : "keyword" - }, - "telemetryRepMdBits" : { - "type" : "keyword" - }, - "telemetryReportVarOptHash" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderRemainingHopCnt" : { - "type" : "keyword" - }, - "IPv6DestIP" : { - "type" : "keyword" - }, - "isDropReport" : { - "type" : "keyword" - }, - "IPvClass" : { - "type" : "keyword" - }, - "telemetryMDLength" : { - "type" : "keyword" - }, - "telemetrySequenceNumber" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderD" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderE" : { - "type" : "keyword" - }, - "telemetryNodeId" : { - "type" : "keyword" - }, - "sha256Hash" : { - "type" : "keyword" - }, - "dataII" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderRes" : { - "type" : "keyword" - } - } - } - } - }, - "aliases" : { } - } diff --git a/playbooks/hcp/templates/udp/udp_data_parsing.json b/playbooks/hcp/templates/udp/udp_data_parsing.json deleted file mode 100644 index a04d5f31..00000000 --- a/playbooks/hcp/templates/udp/udp_data_parsing.json +++ /dev/null @@ -1,1536 +0,0 @@ -{ - "description" : "Transparent Security UDP v4 & v6 data parsing pipeline", - "processors" : [ - { - "set" : { - "field" : "ts.rawData", - "value" : "{{layers.data.data_data_data}}" - } - }, - - { - "script" : { - "lang" : "painless", - "source" : "ctx.ts.telemetryReport = ctx.ts.rawData.substring(0, 48).replace(':','')", - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : "ctx.ts.dropTelemetryReport = ctx.ts.rawData.substring(0, 200).replace(':','')", - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String hardwareIdStr = ctx.layers.data.data_data_data.replace(':','').substring(1, 3); - BigInteger hwdIdBigInt = new BigInteger(hardwareIdStr, 16); - BigInteger bitwiseAnd = new BigInteger('FC', 16); - BigInteger extractedHardwareId = hwdIdBigInt.and(bitwiseAnd); - String hwdIdStr = extractedHardwareId.toString(16); - ctx.ts.telemetryHwdId = hwdIdStr; - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String sequenceNumberPartStrI = tsRawData.substring(2, 3); - String sequenceNumberPartStrII = tsRawData.substring(3, 8); - String sequenceNumberPartStr = tsRawData.substring(2, 8); - BigInteger sequenceNumberBigInt = new BigInteger(sequenceNumberPartStr, 16); - BigInteger bitwiseAnd = new BigInteger('7FFFFFF', 16); - BigInteger updatedSequenceNumber = sequenceNumberBigInt.and(bitwiseAnd); - // String sequenceNumberHexStr = updatedSequenceNumber.toString(16); - ctx.ts.telemetrySequenceNumber = updatedSequenceNumber.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String nodeIdStr = tsRawData.substring(8, 16); - BigInteger nodeIdBigInt = new BigInteger(nodeIdStr, 16); - ctx.ts.telemetryNodeId =nodeIdBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String repTypeStr = tsRawData.substring(16, 17); - BigInteger repTypeBigInt = new BigInteger(repTypeStr, 16); - ctx.ts.telemetryReportType =repTypeBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String inTypeStr = tsRawData.substring(17, 18); - BigInteger inTypeBigInt = new BigInteger(inTypeStr, 16); - ctx.ts.telemetryInType = inTypeBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String reportLengthStr = tsRawData.substring(18, 20); - BigInteger reportLengthBigInt = new BigInteger(reportLengthStr, 16); - ctx.ts.telemetryReportLength = reportLengthBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String mdLengthStr = tsRawData.substring(20, 22); - BigInteger mdLengthBigInt = new BigInteger(mdLengthStr, 16); - ctx.ts.telemetryMDLength = mdLengthBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String dStr = tsRawData.substring(22, 23); - BigInteger dBigInt = new BigInteger(dStr, 16); - BigInteger bitwiseAnd = new BigInteger('8', 16); - BigInteger dRes = dBigInt.and(bitwiseAnd); - ctx.ts.telemetryD = dRes.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String qStr = tsRawData.substring(22, 23); - BigInteger qBigInt = new BigInteger(qStr, 16); - BigInteger bitwiseAnd = new BigInteger('4', 16); - BigInteger qRes = qBigInt.and(bitwiseAnd); - ctx.ts.telemetryQ = qRes.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String fStr = tsRawData.substring(22, 23); - BigInteger fBigInt = new BigInteger(fStr, 16); - BigInteger bitwiseAnd = new BigInteger('2', 16); - BigInteger fRes = fBigInt.and(bitwiseAnd); - ctx.ts.telemetryF = fRes.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String IStr = tsRawData.substring(22, 23); - BigInteger IBigInt = new BigInteger(IStr, 16); - BigInteger bitwiseAnd = new BigInteger('1', 16); - BigInteger IRes = IBigInt.and(bitwiseAnd); - ctx.ts.telemetryI = IRes.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String rsvdStr = tsRawData.substring(23, 24); - BigInteger rsvdBigInt = new BigInteger(rsvdStr, 16); - ctx.ts.telemetryRsvd = rsvdBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String repMdBitsStr = tsRawData.substring(24, 28); - BigInteger repMdBitsBigInt = new BigInteger(repMdBitsStr, 16); - ctx.ts.telemetryRepMdBits = repMdBitsBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String domainSpecificIDStr = tsRawData.substring(28, 32); - BigInteger domainSpecificIDBigInt = new BigInteger(domainSpecificIDStr, 16); - ctx.ts.telemetryDomainSpecificID = domainSpecificIDBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String dsMdBitsStr = tsRawData.substring(32, 36); - BigInteger dsMdBitsStrBigInt = new BigInteger(dsMdBitsStr, 16); - ctx.ts.telemetryDsMdBits = dsMdBitsStrBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String dsMdStatusStr = tsRawData.substring(36, 40); - BigInteger dsMdStatusBigInt = new BigInteger(dsMdStatusStr, 16); - ctx.ts.telemetryDsMdStatus = dsMdStatusBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String varOptStr = tsRawData.substring(40, 48); - BigInteger varOptBigInt = new BigInteger(varOptStr, 16); - ctx.ts.telemetryVarOpt = varOptBigInt.intValue(); - } - - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.isIPv6 = true; - } else - { - ctx.ts.isIPv6 = false; - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - String dropReportStrValue = '2'; - String telemetryIPv4StrValue ='4'; - String telemetryIPv6StrValue ='5'; - String telemetryInTypeStrValue = ctx.ts.telemetryInType.toString(); - if(dropReportStrValue == telemetryInTypeStrValue) - { - ctx.ts.isDropReport = true; - } - if( (telemetryInTypeStrValue == telemetryIPv4StrValue) || (telemetryInTypeStrValue == telemetryIPv6StrValue) ) - { - ctx.ts.isDropReport = false; - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.isIPv4 = true; - } else - { - ctx.ts.isIPv4 = false; - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.data = tsRawData.substring(48, 252); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.data = tsRawData.substring(48, 292); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - ctx.ts.ethernetHeader = tsRawData.substring(48, 76); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - ctx.ts.ethernetHeaderDestMac = tsRawData.substring(48, 60); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - ctx.ts.ethernetHeaderSrcMac = tsRawData.substring(60, 72); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - ctx.ts.ethernetHeaderEtherType = tsRawData.substring(72, 76); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.IPvData = tsRawData.substring(76, 116); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.IPvData = tsRawData.substring(76, 156); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.IPVersion = tsRawData.substring(76, 77); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.IPVersion = tsRawData.substring(76, 77); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String IPv4SrcIPHexString = tsRawData.substring(100, 108); - String IPv4SrcIP = ""; - for(int i = 0; i < IPv4SrcIPHexString.length(); i = i + 2) { - IPv4SrcIP = IPv4SrcIP + Integer.valueOf(IPv4SrcIPHexString.substring(i, i+2), 16) + "."; - } - IPv4SrcIP = IPv4SrcIP.substring(0, IPv4SrcIP.length()-1); - ctx.ts.IPv4SrcIP = IPv4SrcIP; - } - if( etherType == '86dd' && ipVersion == '6') - { - String IPv6SrcIPHexString = tsRawData.substring(92,124); - String IPv6SrcIP = ""; - for(int i = 0; i < IPv6SrcIPHexString.length(); i = i + 4) { - IPv6SrcIP = IPv6SrcIP + IPv6SrcIPHexString.substring(i, i+4) + ":"; - } - IPv6SrcIP = IPv6SrcIP.substring(0, IPv6SrcIP.length()-1); - ctx.ts.IPv6SrcIP = IPv6SrcIP; - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String IPv4DestIPHexString = tsRawData.substring(108, 116); - String IPv4DestIP = ""; - for(int i = 0; i < IPv4DestIPHexString.length(); i = i + 2) { - IPv4DestIP = IPv4DestIP + Integer.valueOf(IPv4DestIPHexString.substring(i, i+2), 16) + "."; - } - IPv4DestIP = IPv4DestIP.substring(0, IPv4DestIP.length()-1); - ctx.ts.IPv4DestIP = IPv4DestIP; - } - if( etherType == '86dd' && ipVersion == '6') - { - String IPv6DestIPHexString = tsRawData.substring(124,156); - String IPv6DestIP = ""; - for(int i = 0; i < IPv6DestIPHexString.length(); i = i + 4) { - IPv6DestIP = IPv6DestIP + IPv6DestIPHexString.substring(i, i+4) + ":"; - } - IPv6DestIP = IPv6DestIP.substring(0, IPv6DestIP.length()-1); - ctx.ts.IPv6DestIP = IPv6DestIP; - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDGPData = tsRawData.substring(116, 132); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDGPData = tsRawData.substring(156, 172); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDGSrcPortStr = tsRawData.substring(116, 120); - BigInteger UDGSrcPortBigInt = new BigInteger(UDGSrcPortStr, 16); - ctx.ts.UDGSrcPort = UDGSrcPortBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDGSrcPortStr = tsRawData.substring(156, 160); - BigInteger UDGSrcPortBigInt = new BigInteger(UDGSrcPortStr, 16); - ctx.ts.UDGSrcPort = UDGSrcPortBigInt.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDGDstPortStr = tsRawData.substring(120, 124); - BigInteger UDGDstPortBigInt = new BigInteger(UDGDstPortStr, 16); - ctx.ts.UDGDstPort = UDGDstPortBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDGDstPortStr = tsRawData.substring(160, 164); - BigInteger UDGDstPortBigInt = new BigInteger(UDGDstPortStr, 16); - ctx.ts.UDGDstPort = UDGDstPortBigInt.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDGLengthStr = tsRawData.substring(124, 128); - BigInteger UDGLengthBigInt = new BigInteger(UDGLengthStr, 16); - ctx.ts.UDGLength = UDGLengthBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDGLengthStr = tsRawData.substring(164, 168); - BigInteger UDGLengthBigInt = new BigInteger(UDGLengthStr, 16); - ctx.ts.UDGLength = UDGLengthBigInt.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDGChecksumStr = tsRawData.substring(128, 132); - BigInteger UDGChecksumBigInt = new BigInteger(UDGChecksumStr, 16); - ctx.ts.UDGChecksum = UDGChecksumBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDGChecksumStr = tsRawData.substring(168, 172); - BigInteger UDGChecksumBigInt = new BigInteger(UDGChecksumStr, 16); - ctx.ts.UDGChecksum = UDGChecksumBigInt.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.InBandNetworkTelemetryData = tsRawData.substring(132, 196); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.InBandNetworkTelemetryData = tsRawData.substring(172, 236); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDPIntShimHeaderData = tsRawData.substring(132, 140); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDPIntShimHeaderData = tsRawData.substring(172, 180); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDPIntShimHeaderType = tsRawData.substring(132, 133); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDPIntShimHeaderType = tsRawData.substring(172, 173); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String udpIntShimNPTStr = tsRawData.substring(133, 134); - BigInteger udpIntShimNPTBigInt = new BigInteger(udpIntShimNPTStr, 16); - BigInteger bitwiseAnd = new BigInteger('C', 16); - BigInteger result = udpIntShimNPTBigInt.and(bitwiseAnd); - ctx.ts.UDPIntShimHeaderNPT = result.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String udpIntShimNPTStr = tsRawData.substring(173, 174); - BigInteger udpIntShimNPTBigInt = new BigInteger(udpIntShimNPTStr, 16); - BigInteger bitwiseAnd = new BigInteger('C', 16); - BigInteger result = udpIntShimNPTBigInt.and(bitwiseAnd); - ctx.ts.UDPIntShimHeaderNPT = result.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDPIntShimHeaderRes1 = tsRawData.substring(133, 134); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDPIntShimHeaderRes1 = tsRawData.substring(174, 175); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String lengthStr = tsRawData.substring(134, 136); - BigInteger lengthStrBigInt = new BigInteger(lengthStr, 16); - ctx.ts.UDPIntShimHeaderLength= lengthStrBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String lengthStr = tsRawData.substring(174, 176); - BigInteger lengthStrBigInt = new BigInteger(lengthStr, 16); - ctx.ts.UDPIntShimHeaderLength = lengthStrBigInt.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDPIntShimHeaderRes2 = tsRawData.substring(136, 138); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDPIntShimHeaderRes2 = tsRawData.substring(176, 178); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String nextProtoStr = tsRawData.substring(138, 140); - BigInteger nextProtoBigInt = new BigInteger(nextProtoStr, 16); - ctx.ts.UDPIntShimHeaderNextProto = nextProtoBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String nextProtoStr = tsRawData.substring(178, 180); - BigInteger nextProtoBigInt = new BigInteger(nextProtoStr, 16); - ctx.ts.UDPIntShimHeaderNextProto = nextProtoBigInt.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.INTMetadataHeaderData = tsRawData.substring(140, 164); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.INTMetadataHeaderData = tsRawData.substring(180, 204); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderVersionStr = tsRawData.substring(140, 141); - BigInteger UDPIntMetaDataHeaderVersionStrBigInt = new BigInteger(UDPIntMetaDataHeaderVersionStr, 16); - ctx.ts.UDPIntMetaDataHeaderVersion = UDPIntMetaDataHeaderVersionStrBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderVersionStr = tsRawData.substring(180, 181); - BigInteger UDPIntMetaDataHeaderVersionStrBigInt = new BigInteger(UDPIntMetaDataHeaderVersionStr, 16); - ctx.ts.UDPIntMetaDataHeaderVersion = UDPIntMetaDataHeaderVersionStrBigInt.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderRes = tsRawData.substring(141, 142); - BigInteger UDPIntMetaDataHeaderResBigInt = new BigInteger(UDPIntMetaDataHeaderRes, 16); - BigInteger bitwiseAnd = new BigInteger('C', 16); - BigInteger result = UDPIntMetaDataHeaderResBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderRes = result.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderRes = tsRawData.substring(181, 182); - BigInteger UDPIntMetaDataHeaderResBigInt = new BigInteger(UDPIntMetaDataHeaderRes, 16); - BigInteger bitwiseAnd = new BigInteger('C', 16); - BigInteger result = UDPIntMetaDataHeaderResBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderRes = result.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderD = tsRawData.substring(141, 142); - BigInteger UDPIntMetaDataHeaderDBigInt = new BigInteger(UDPIntMetaDataHeaderD, 16); - BigInteger bitwiseAnd = new BigInteger('2', 16); - BigInteger result = UDPIntMetaDataHeaderDBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderD = result.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderD = tsRawData.substring(181, 182); - BigInteger UDPIntMetaDataHeaderDBigInt = new BigInteger(UDPIntMetaDataHeaderD, 16); - BigInteger bitwiseAnd = new BigInteger('2', 16); - BigInteger result = UDPIntMetaDataHeaderDBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderD = result.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderE = tsRawData.substring(141, 142); - BigInteger UDPIntMetaDataHeaderEBigInt = new BigInteger(UDPIntMetaDataHeaderE, 16); - BigInteger bitwiseAnd = new BigInteger('1', 16); - BigInteger result = UDPIntMetaDataHeaderEBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderE = result.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderE = tsRawData.substring(181, 182); - BigInteger UDPIntMetaDataHeaderEBigInt = new BigInteger(UDPIntMetaDataHeaderE, 16); - BigInteger bitwiseAnd = new BigInteger('1', 16); - BigInteger result = UDPIntMetaDataHeaderEBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderE = result.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderM = tsRawData.substring(142, 143); - BigInteger UDPIntMetaDataHeaderMBigInt = new BigInteger(UDPIntMetaDataHeaderM, 16); - BigInteger bitwiseAnd = new BigInteger('8', 16); - BigInteger result = UDPIntMetaDataHeaderMBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderM = result.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderM = tsRawData.substring(182, 183); - BigInteger UDPIntMetaDataHeaderMBigInt = new BigInteger(UDPIntMetaDataHeaderM, 16); - BigInteger bitwiseAnd = new BigInteger('8', 16); - BigInteger result = UDPIntMetaDataHeaderMBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderM = result.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderPerhopMetadataLengthStr = tsRawData.substring(144, 146); - BigInteger UDPIntMetaDataHeaderPerhopMetadataLengthBigInt = new BigInteger(UDPIntMetaDataHeaderPerhopMetadataLengthStr, 16); - BigInteger bitwiseAnd = new BigInteger('1F', 16); - BigInteger result = UDPIntMetaDataHeaderPerhopMetadataLengthBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderPerhopMetadataLength = result.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderPerhopMetadataLengthStr = tsRawData.substring(184, 186); - BigInteger UDPIntMetaDataHeaderPerhopMetadataLengthBigInt = new BigInteger(UDPIntMetaDataHeaderPerhopMetadataLengthStr, 16); - BigInteger bitwiseAnd = new BigInteger('1F', 16); - BigInteger result = UDPIntMetaDataHeaderPerhopMetadataLengthBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderPerhopMetadataLength = result.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderRemainingHopCntStr = tsRawData.substring(146, 148); - BigInteger UDPIntMetaDataHeaderRemainingHopCntBigInt = new BigInteger(UDPIntMetaDataHeaderRemainingHopCntStr, 16); - ctx.ts.UDPIntMetaDataHeaderRemainingHopCnt = UDPIntMetaDataHeaderRemainingHopCntBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderRemainingHopCntStr = tsRawData.substring(186, 188); - BigInteger UDPIntMetaDataHeaderRemainingHopCntBigInt = new BigInteger(UDPIntMetaDataHeaderRemainingHopCntStr, 16); - ctx.ts.UDPIntMetaDataHeaderRemainingHopCnt = UDPIntMetaDataHeaderRemainingHopCntBigInt.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderInstructionBitmapStr = tsRawData.substring(148, 152); - BigInteger UDPIntMetaDataHeaderInstructionBitmapBigInt = new BigInteger(UDPIntMetaDataHeaderInstructionBitmapStr, 16); - ctx.ts.UDPIntMetaDataHeaderInstructionBitmap = UDPIntMetaDataHeaderInstructionBitmapBigInt.toString(16); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderInstructionBitmapStr = tsRawData.substring(188, 192); - BigInteger UDPIntMetaDataHeaderInstructionBitmapBigInt = new BigInteger(UDPIntMetaDataHeaderInstructionBitmapStr, 16); - ctx.ts.UDPIntMetaDataHeaderInstructionBitmap = UDPIntMetaDataHeaderInstructionBitmapBigInt.toString(16); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderDomainSpecificIDStr = tsRawData.substring(152, 156); - BigInteger UDPIntMetaDataHeaderDomainSpecificIDBigInt = new BigInteger(UDPIntMetaDataHeaderDomainSpecificIDStr, 16); - ctx.ts.UDPIntMetaDataHeaderDomainSpecificID = UDPIntMetaDataHeaderDomainSpecificIDBigInt.toString(16); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderDomainSpecificIDStr = tsRawData.substring(192, 196); - BigInteger UDPIntMetaDataHeaderDomainSpecificIDBigInt = new BigInteger(UDPIntMetaDataHeaderDomainSpecificIDStr, 16); - ctx.ts.UDPIntMetaDataHeaderDomainSpecificID = UDPIntMetaDataHeaderDomainSpecificIDBigInt.toString(16); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderDSInstructionStr = tsRawData.substring(156, 160); - BigInteger UDPIntMetaDataHeaderDSInstructionBigInt = new BigInteger(UDPIntMetaDataHeaderDSInstructionStr, 16); - ctx.ts.UDPIntMetaDataHeaderDSInstruction = UDPIntMetaDataHeaderDSInstructionBigInt.toString(16); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderDSInstructionStr = tsRawData.substring(196, 200); - BigInteger UDPIntMetaDataHeaderDSInstructionBigInt = new BigInteger(UDPIntMetaDataHeaderDSInstructionStr, 16); - ctx.ts.UDPIntMetaDataHeaderDSInstruction = UDPIntMetaDataHeaderDSInstructionBigInt.toString(16); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderDSFlagsStr = tsRawData.substring(160, 164); - BigInteger UDPIntMetaDataHeaderDSFlagsBigInt = new BigInteger(UDPIntMetaDataHeaderDSFlagsStr, 16); - ctx.ts.UDPIntMetaDataHeaderDSFlags = UDPIntMetaDataHeaderDSFlagsBigInt.toString(16); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderDSFlagsStr = tsRawData.substring(200, 204); - BigInteger UDPIntMetaDataHeaderDSFlagsBigInt = new BigInteger(UDPIntMetaDataHeaderDSFlagsStr, 16); - ctx.ts.UDPIntMetaDataHeaderDSFlags = UDPIntMetaDataHeaderDSFlagsBigInt.toString(16); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.INTMetadataStackData = tsRawData.substring(164, 196); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.INTMetadataStackData = tsRawData.substring(204, 236); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.INTMetadataStackSwitchID = tsRawData.substring(164, 172); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.INTMetadataStackSwitchID = tsRawData.substring(204, 212); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.INTMetadataSourceMetadata = tsRawData.substring(172, 196); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.INTMetadataSourceMetadata = tsRawData.substring(212, 236); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.INTMetadataSourceMetadataSwitchId = tsRawData.substring(172, 180); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.INTMetadataSourceMetadataSwitchId = tsRawData.substring(212, 220); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String rawOriginatingMac = tsRawData.substring(180, 192); - StringBuilder formattedOriginatingMac = new StringBuilder(); - for(int i = 0; i < rawOriginatingMac.length();) - { - formattedOriginatingMac.append(rawOriginatingMac.substring(i++, ++i)); - if(i < rawOriginatingMac.length()) formattedOriginatingMac.append(":"); - } - ctx.ts.INTMetadataSourceMetadataOriginatingMac = formattedOriginatingMac.toString(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String rawOriginatingMac = tsRawData.substring(220, 232); - StringBuilder formattedOriginatingMac = new StringBuilder(); - for(int i = 0; i < rawOriginatingMac.length();) - { - formattedOriginatingMac.append(rawOriginatingMac.substring(i++, ++i)); - if(i < rawOriginatingMac.length()) formattedOriginatingMac.append(":"); - } - ctx.ts.INTMetadataSourceMetadataOriginatingMac = formattedOriginatingMac.toString(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.INTMetadataSourceMetadataReserved = tsRawData.substring(192, 196); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.INTMetadataSourceMetadataReserved = tsRawData.substring(232, 236); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDP2Data = tsRawData.substring(196, 212); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDP2Data = tsRawData.substring(236, 252); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDP2SrcPortStr = tsRawData.substring(196, 200); - BigInteger UDP2SrcPortBigInt = new BigInteger(UDP2SrcPortStr,16); - ctx.ts.UDP2SrcPort = UDP2SrcPortBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDP2SrcPortStr = tsRawData.substring(236, 240); - BigInteger UDP2SrcPortBigInt = new BigInteger(UDP2SrcPortStr,16); - ctx.ts.UDP2SrcPort = UDP2SrcPortBigInt.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDP2DstPortStr = tsRawData.substring(200, 204); - BigInteger UDP2DstPortBigInt = new BigInteger(UDP2DstPortStr,16); - ctx.ts.UDP2DstPort = UDP2DstPortBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDP2DstPortStr = tsRawData.substring(240, 244); - BigInteger UDP2DstPortBigInt = new BigInteger(UDP2DstPortStr,16); - ctx.ts.UDP2DstPort = UDP2DstPortBigInt.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDP2Length = tsRawData.substring(204, 208); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDP2Length = tsRawData.substring(244, 248); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDP2Checksum = tsRawData.substring(208, 212); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDP2Checksum = tsRawData.substring(248, 252); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.dataII = tsRawData.substring(212, 252); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.dataII = tsRawData.substring(252, 292); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == true) - { - String dropTelemetryReport = ctx.ts.dropTelemetryReport; - String telemetryReportVarOpBsmdStr = dropTelemetryReport.substring(40,48); - BigInteger telemetryReportVarOpBsmdBigInt = new BigInteger(telemetryReportVarOpBsmdStr, 16); - ctx.ts.telemetryReportVarOpBsmd = telemetryReportVarOpBsmdBigInt.intValue(); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == true) - { - String dropTelemetryReport = ctx.ts.dropTelemetryReport; - String telemetryReportVarOptTimeStampStr = dropTelemetryReport.substring(48,56); - BigInteger telemetryReportVarOptTimeStampBigInt = new BigInteger(telemetryReportVarOptTimeStampStr, 16); - ctx.ts.telemetryReportVarOptTimeStamp = telemetryReportVarOptTimeStampBigInt.intValue(); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == true) - { - String dropTelemetryReport = ctx.ts.dropTelemetryReport; - String telemetryReportVarOptDropCountStr = dropTelemetryReport.substring(56,64); - BigInteger telemetryReportVarOptDropCountBigInt = new BigInteger(telemetryReportVarOptDropCountStr, 16); - ctx.ts.telemetryReportVarOptDropCount = telemetryReportVarOptDropCountBigInt.intValue(); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == true) - { - String dropTelemetryReport = ctx.ts.dropTelemetryReport; - String telemetryReportVarOptHashStr = dropTelemetryReport.substring(64,96); - BigInteger telemetryReportVarOptHashBigInt = new BigInteger(telemetryReportVarOptHashStr, 16); - ctx.ts.telemetryReportVarOptHash = telemetryReportVarOptHashBigInt; - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String macAddress = ctx.ts.INTMetadataSourceMetadataOriginatingMac; - String destinationPort = ctx.ts.UDP2DstPort.toString(); - String IPvDestIP = ctx.ts.IPv4DestIP; - String combinedKeys = ''; - if(ctx.ts.isIPv4 == true) - { - combinedKeys = combinedKeys.concat(macAddress).concat('|').concat(destinationPort).concat('|').concat(IPvDestIP).concat('|').concat('::'); - ctx.ts.dropHashInput = combinedKeys; - } - if(ctx.ts.isIPv6 == true) - { - combinedKeys = combinedKeys.concat(macAddress).concat('|').concat(destinationPort).concat('|').concat(IPvDestIP).concat('|').concat('::'); - ctx.ts.dropHashInput = combinedKeys; - } - } - """, - "ignore_failure" : false - } - }, - - { - "script" : { - "ignore_failure" : false, - "id" : "calculateKeyHash" - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String completeSHA256Hash = ctx.ts.sha256Hash; - String subsetSHA256HashStr= completeSHA256Hash.substring(0,16); - BigInteger hashBigInt = new BigInteger(subsetSHA256HashStr, 16); - ctx.ts.dropReportHash = hashBigInt; - """, - "ignore_failure" : true - } - } - ] - } diff --git a/playbooks/hcp/templates/udp/udp_monitor_def.json b/playbooks/hcp/templates/udp/udp_monitor_def.json deleted file mode 100644 index 5733cf77..00000000 --- a/playbooks/hcp/templates/udp/udp_monitor_def.json +++ /dev/null @@ -1,166 +0,0 @@ -{ - "type": "monitor", - "name": "DDOS-UDP-Monitor", - "enabled": true, - "schedule": { - "period": { - "interval": 1, - "unit": "MINUTES" - } - }, - "inputs": [{ - "search": { - "indices": ["packets-*"], - "query": { - "size": 0, - "query": { - "bool": { - "filter": [ - { - "range": { - "timestamp": { - "from": "{{period_end}}||-1m", - "to": "{{period_end}}", - "include_lower": true, - "include_upper": true, - "format": "epoch_millis", - "boost": 1 - } - } - }, - { - "wildcard": { - "ts.UDP2DstPort": { - "wildcard": "*5792*", - "boost": 1 - } - } - } - ], - "adjust_pure_negative": true, - "boost": 1 - } - }, - "_source": { - "includes": [ - "ts.UDP2DstPort", - "ts.IPvDestAddr", - "ts.INTMetadataSourceMetadataOriginatingMac", - "ts.IPv4SrcIP", - "ts.IPv4DestIP", - "ts.IPv6SrcIP", - "ts.IPv6DestIP" - ], - "excludes": [] - }, - "aggregations": { - "UDP2DstPortValueCount": { - "value_count": { - "field": "ts.UDP2DstPort" - } - }, - "IPvDestAddrValueCount": { - "value_count": { - "field": "ts.IPvDestAddr" - } - }, - "INTMetadataSourceMetadataOriginatingMacValueCount": { - "value_count": { - "field": "ts.INTMetadataSourceMetadataOriginatingMac" - } - }, - "OriginatingMacAddrUDPv4": { - "terms": { - "field": "ts.INTMetadataSourceMetadataOriginatingMac", - "size": 10, - "min_doc_count": 1, - "shard_min_doc_count": 0, - "show_term_doc_count_error": false, - "order": [ - { - "_count": "desc" - }, - { - "_key": "asc" - } - ] - } - }, - "IPv4SrcIP": { - "terms": { - "field": "ts.IPv4SrcIP", - "size": 10, - "min_doc_count": 1, - "shard_min_doc_count": 0, - "show_term_doc_count_error": false, - "order": [ - { - "_count": "desc" - }, - { - "_key": "asc" - } - ] - } - }, - "IPv4DestIP": { - "terms": { - "field": "ts.IPv4DestIP", - "size": 10, - "min_doc_count": 1, - "shard_min_doc_count": 0, - "show_term_doc_count_error": false, - "order": [ - { - "_count": "desc" - }, - { - "_key": "asc" - } - ] - } - }, - "UDP2DstPortValue": { - "terms": { - "field": "ts.UDP2DstPort", - "size": 10, - "min_doc_count": 1, - "shard_min_doc_count": 0, - "show_term_doc_count_error": false, - "order": [ - { - "_count": "desc" - }, - { - "_key": "asc" - } - ] - } - } - } - } - } - }], - "triggers": [{ - "name": "DDOS-Trigger", - "severity": "1", - "condition": { - "script": { - "source": "if(ctx.results[0].hits.total.value > 0){ctx.results[0].aggregations.UDP2DstPortValueCount.value >= 10}", - "lang": "painless" - } - }, - "actions": [{ - "name": "Sdn-WebHook-Action", - "destination_id": "sdnWebhookResponseId", - "message_template": { - "source":"{\"event_action\":\"trigger\", \"dst_ip\":\"{{ctx.results.0.aggregations.IPv4DestIP.buckets.0.key}}\",\"dst_port\":\"{{ctx.results.0.aggregations.UDP2DstPortValue.buckets.0.key}}\",\"src_ip\":\"{{ctx.results.0.aggregations.IPv4SrcIP.buckets.0.key}}\",\"src_mac\":\"{{ctx.results.0.aggregations.OriginatingMacAddrUDPv4.buckets.0.key}}\"}", - "lang": "mustache", - "options": { - "content_type": "application/json" - } - }, - "throttle_enabled": false - }] - }] -} diff --git a/playbooks/hcp/templates/udp_and_tcp_data_mapping.json b/playbooks/hcp/templates/udp_and_tcp_data_mapping.json deleted file mode 100644 index 7bc5bc80..00000000 --- a/playbooks/hcp/templates/udp_and_tcp_data_mapping.json +++ /dev/null @@ -1,320 +0,0 @@ -{ - "index_patterns" : [ - "packets-*" - ], - "settings" : { - "index" : { - "number_of_replicas" : "0", - "default_pipeline" : "ts_parsing", - "max_script_fields" : "200" - } - }, - "mappings" : { - "dynamic" : "false", - "properties" : { - "layers" : { - "properties" : { - "data_raw" : { - "type" : "keyword" - }, - "data" : { - "properties" : { - "data_data_data_raw" : { - "type" : "keyword" - }, - "data_data_len" : { - "type" : "keyword" - }, - "data_data_data" : { - "type" : "keyword" - } - } - } - } - }, - "timestamp" : { - "type" : "date" - }, - "ts" : { - "properties" : { - "INTMetadataSourceMetadataReserved" : { - "type" : "keyword" - }, - "UDPIntShimHeaderType" : { - "type" : "keyword" - }, - "telemetryReportLength" : { - "type" : "keyword" - }, - "data" : { - "type" : "keyword" - }, - "ethernetHeaderDestMac" : { - "type" : "keyword" - }, - "telemetryRsvd" : { - "type" : "keyword" - }, - "UDGSrcPort" : { - "type" : "keyword" - }, - "INTMetadataSourceMetadataSwitchId" : { - "type" : "keyword" - }, - "ethernetHeaderSrcMac" : { - "type" : "keyword" - }, - "UDPIntShimHeaderNextProto" : { - "type" : "keyword" - }, - "isIPv4" : { - "type" : "keyword" - }, - "isTCP" : { - "type" : "keyword" - }, - "isTCPv4" : { - "type" : "keyword" - }, - "isTCPv6" : { - "type" : "keyword" - }, - "isUDP" : { - "type" : "keyword" - }, - "isUDPv4" : { - "type" : "keyword" - }, - "isUDPv6" : { - "type" : "keyword" - }, - "dropReportHash": { - "type": "keyword" - }, - "sha256Hash": { - "type": "keyword" - }, - "nextProto": { - "type": "keyword" - }, - "TCPData" : { - "type" : "keyword" - }, - "TCPSrcPort" : { - "type" : "keyword" - }, - "TCPDestPort" : { - "type" : "keyword" - }, - "TCPIntShimHeaderNextProto" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderVersion" : { - "type" : "keyword" - }, - "IPvNextHdrProto" : { - "type" : "keyword" - }, - "telemetryReportVersion" : { - "type" : "keyword" - }, - "isIPv6" : { - "type" : "keyword" - }, - "IPv4SrcIP": { - "type" : "keyword" - }, - "UDP2DstPort": { - "type": "keyword" - }, - "UDP2Checksum": { - "type": "keyword" - }, - "UDP2Length": { - "type": "keyword" - }, - "combinedHashKeys": { - "type": "keyword" - }, - "UDP2SrcPort": { - "type": "keyword" - }, - "IPv6SrcIP": { - "type" : "keyword" - }, - "IPv4DestIP":{ - "type" : "keyword" - }, - "IPv6DestIP":{ - "type" : "keyword" - }, - "INTMetadataSourceMetadataOriginatingMac" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderInstructionBitmap" : { - "type" : "keyword" - }, - "INTMetadataSourceMetadata" : { - "type" : "keyword" - }, - "IPvSrcAddr" : { - "type" : "keyword" - }, - "tsRawDataSize" : { - "type" : "keyword" - }, - "UDGPData" : { - "type" : "keyword" - }, - "UDGChecksum" : { - "type" : "keyword" - }, - "INTMetadataHeaderData" : { - "type" : "keyword" - }, - "IPvDestAddr" : { - "type" : "keyword" - }, - "telemetryInType" : { - "type" : "keyword" - }, - "telemetryReport" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderDSFlags" : { - "type" : "keyword" - }, - "telemetryDsMdStatus" : { - "type" : "keyword" - }, - "InBandNetworkTelemetryData" : { - "type" : "keyword" - }, - "UDPIntShimHeaderRes2" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderDSInstruction" : { - "type" : "keyword" - }, - "UDPIntShimHeaderRes1" : { - "type" : "keyword" - }, - "UDP2Data" : { - "type" : "keyword" - }, - "telemetryHwdId" : { - "type" : "keyword" - }, - "telemetryVarOpt" : { - "type" : "keyword" - }, - "IPvData" : { - "type" : "keyword" - }, - "ethernetHeaderEtherType" : { - "type" : "keyword" - }, - "INTMetadataStackData" : { - "type" : "keyword" - }, - "IPvFlow" : { - "type" : "keyword" - }, - "UDPIntShimHeaderData" : { - "type" : "keyword" - }, - "INTMetadataStackSwitchID" : { - "type" : "keyword" - }, - "IPVersion" : { - "type" : "keyword" - }, - "IPvPayloadLength" : { - "type" : "keyword" - }, - "UDPIntShimHeaderLength" : { - "type" : "keyword" - }, - "UDPIntShimHeaderNPT" : { - "type" : "keyword" - }, - "UDGLength" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderM" : { - "type" : "keyword" - }, - "telemetryD" : { - "type" : "keyword" - }, - "rawData" : { - "type" : "keyword" - }, - "telemetryF" : { - "type" : "keyword" - }, - "telemetryReportType" : { - "type" : "keyword" - }, - "telemetryI" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderPerhopMetadataLength" : { - "type" : "keyword" - }, - "telemetryQ" : { - "type" : "keyword" - }, - "telemetryDomainSpecificID" : { - "type" : "keyword" - }, - "udpLayersSize" : { - "type" : "keyword" - }, - "UDGDstPort" : { - "type" : "keyword" - }, - "IPvNextHopLimit" : { - "type" : "keyword" - }, - "telemetryDsMdBits" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderDomainSpecificID" : { - "type" : "keyword" - }, - "telemetryRepMdBits" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderRemainingHopCnt" : { - "type" : "keyword" - }, - "telemetrySequenceNumber" : { - "type" : "keyword" - }, - "telemetryMDLength" : { - "type" : "keyword" - }, - "IPvClass" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderD" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderE" : { - "type" : "keyword" - }, - "telemetryNodeId" : { - "type" : "keyword" - }, - "dataII" : { - "type" : "keyword" - }, - "UDPIntMetaDataHeaderRes" : { - "type" : "keyword" - } - } - } - } - } -} diff --git a/playbooks/hcp/templates/udp_and_tcp_data_parsing.json b/playbooks/hcp/templates/udp_and_tcp_data_parsing.json deleted file mode 100644 index e1ff806e..00000000 --- a/playbooks/hcp/templates/udp_and_tcp_data_parsing.json +++ /dev/null @@ -1,1777 +0,0 @@ -{ - "description" : "Transparent Security TCP & UDP v4 & v6 data parsing pipeline", - "processors" : [ - { - "set" : { - "field" : "ts.rawData", - "value" : "{{layers.data.data_data_data}}" - } - }, - { - "script" : { - "lang" : "painless", - "source" : "ctx.ts.telemetryReport =ctx.ts.rawData.substring(0, 48).replace(':','')", - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : "ctx.ts.dropTelemetryReport = ctx.ts.rawData.substring(0, 180).replace(':','')", - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String hardwareIdStr = ctx.layers.data.data_data_data.replace(':','').substring(1, 3); - BigInteger hwdIdBigInt = new BigInteger(hardwareIdStr, 16); - BigInteger bitwiseAnd = new BigInteger('FC', 16); - BigInteger extractedHardwareId = hwdIdBigInt.and(bitwiseAnd); - String hwdIdStr = extractedHardwareId.toString(16); - ctx.ts.telemetryHwdId = hwdIdStr; - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String sequenceNumberPartStrI = tsRawData.substring(2, 3); - String sequenceNumberPartStrII = tsRawData.substring(3, 8); - String sequenceNumberPartStr = tsRawData.substring(2, 8); - BigInteger sequenceNumberBigInt = new BigInteger(sequenceNumberPartStr, 16); - BigInteger bitwiseAnd = new BigInteger('7FFFFFF', 16); - BigInteger updatedSequenceNumber = sequenceNumberBigInt.and(bitwiseAnd); - // String sequenceNumberHexStr = updatedSequenceNumber.toString(16); - ctx.ts.telemetrySequenceNumber = updatedSequenceNumber.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String nodeIdStr = tsRawData.substring(8, 16); - BigInteger nodeIdBigInt = new BigInteger(nodeIdStr, 16); - ctx.ts.telemetryNodeId =nodeIdBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String repTypeStr = tsRawData.substring(16, 17); - BigInteger repTypeBigInt = new BigInteger(repTypeStr, 16); - ctx.ts.telemetryReportType =repTypeBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String inTypeStr = tsRawData.substring(17, 18); - BigInteger inTypeBigInt = new BigInteger(inTypeStr, 16); - ctx.ts.telemetryInType = inTypeBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String reportLengthStr = tsRawData.substring(18, 20); - BigInteger reportLengthBigInt = new BigInteger(reportLengthStr, 16); - ctx.ts.telemetryReportLength = reportLengthBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String mdLengthStr = tsRawData.substring(20, 22); - BigInteger mdLengthBigInt = new BigInteger(mdLengthStr, 16); - ctx.ts.telemetryMDLength = mdLengthBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String dStr = tsRawData.substring(22, 23); - BigInteger dBigInt = new BigInteger(dStr, 16); - BigInteger bitwiseAnd = new BigInteger('8', 16); - BigInteger dRes = dBigInt.and(bitwiseAnd); - ctx.ts.telemetryD = dRes.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String qStr = tsRawData.substring(22, 23); - BigInteger qBigInt = new BigInteger(qStr, 16); - BigInteger bitwiseAnd = new BigInteger('4', 16); - BigInteger qRes = qBigInt.and(bitwiseAnd); - ctx.ts.telemetryQ = qRes.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String fStr = tsRawData.substring(22, 23); - BigInteger fBigInt = new BigInteger(fStr, 16); - BigInteger bitwiseAnd = new BigInteger('2', 16); - BigInteger fRes = fBigInt.and(bitwiseAnd); - ctx.ts.telemetryF = fRes.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String IStr = tsRawData.substring(22, 23); - BigInteger IBigInt = new BigInteger(IStr, 16); - BigInteger bitwiseAnd = new BigInteger('1', 16); - BigInteger IRes = IBigInt.and(bitwiseAnd); - ctx.ts.telemetryI = IRes.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String rsvdStr = tsRawData.substring(23, 24); - BigInteger rsvdBigInt = new BigInteger(rsvdStr, 16); - ctx.ts.telemetryRsvd = rsvdBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String repMdBitsStr = tsRawData.substring(24, 28); - BigInteger repMdBitsBigInt = new BigInteger(repMdBitsStr, 16); - ctx.ts.telemetryRepMdBits = repMdBitsBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String domainSpecificIDStr = tsRawData.substring(28, 32); - BigInteger domainSpecificIDBigInt = new BigInteger(domainSpecificIDStr, 16); - ctx.ts.telemetryDomainSpecificID = domainSpecificIDBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String dsMdBitsStr = tsRawData.substring(32, 36); - BigInteger dsMdBitsStrBigInt = new BigInteger(dsMdBitsStr, 16); - ctx.ts.telemetryDsMdBits = dsMdBitsStrBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String dsMdStatusStr = tsRawData.substring(36, 40); - BigInteger dsMdStatusBigInt = new BigInteger(dsMdStatusStr, 16); - ctx.ts.telemetryDsMdStatus = dsMdStatusBigInt.intValue(); - """, - "ignore_failure" : true - } - }, - { - "script" : { - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String varOptStr = tsRawData.substring(40, 48); - BigInteger varOptBigInt = new BigInteger(varOptStr, 16); - ctx.ts.telemetryVarOpt = varOptBigInt.intValue(); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.isIPv6 = true; - } else - { - ctx.ts.isIPv6 = false; - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.isIPv4 = true; - } else - { - ctx.ts.isIPv4 = false; - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - String dropReportStrValue = '2'; - String telemetryIPv4StrValue ='4'; - String telemetryIPv6StrValue ='5'; - String telemetryInTypeStrValue = ctx.ts.telemetryInType.toString(); - if(dropReportStrValue == telemetryInTypeStrValue) - { - ctx.ts.isDropReport = true; - } - if( (telemetryInTypeStrValue == telemetryIPv4StrValue) || (telemetryInTypeStrValue == telemetryIPv6StrValue) ) - { - ctx.ts.isDropReport = false; - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String nextProtoStr = tsRawData.substring(138, 140); - BigInteger nextProtoBigInt = new BigInteger(nextProtoStr, 16); - ctx.ts.nextProto = nextProtoBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String nextProtoStr = tsRawData.substring(178, 180); - BigInteger nextProtoBigInt = new BigInteger(nextProtoStr, 16); - ctx.ts.nextProto = nextProtoBigInt.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - String tcpNextProtoStr = '6'; - int tcpNextProtoInt = Integer.parseInt(tcpNextProtoStr); - - if( etherType == '0800' && ipVersion == '4') - { - - if( ctx.ts.nextProto == tcpNextProtoInt ) - { - ctx.ts.isTCPv4 = true; - }else { - ctx.ts.isTCPv4 = false; - } - } - - if( etherType == '86dd' && ipVersion == '6') - { - - if( ctx.ts.nextProto == tcpNextProtoInt ) - { - ctx.ts.isTCPv6 = true; - }else { - ctx.ts.isTCPv6 = false; - } - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - String udpNextProtoStr = '17'; - int udpNextProtoInt = Integer.parseInt(udpNextProtoStr); - if( etherType == '0800' && ipVersion == '4') - { - - if(ctx.ts.nextProto == udpNextProtoInt ) - { - ctx.ts.isUDPv4 = true; - }else { - ctx.ts.isUDPv4 = false; - } - } - - if( etherType == '86dd' && ipVersion == '6') - { - String nextProtoStr = tsRawData.substring(178, 180); - BigInteger nextProtoBigInt = new BigInteger(nextProtoStr, 16); - - if(ctx.ts.nextProto == udpNextProtoInt ) - { - ctx.ts.isUDPv6 = true; - }else { - ctx.ts.isUDPv6 = false; - } - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.data = tsRawData.substring(48, 252); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.data = tsRawData.substring(48, 292); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - ctx.ts.ethernetHeader = tsRawData.substring(48, 76); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - ctx.ts.ethernetHeaderDestMac = tsRawData.substring(48, 60); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - ctx.ts.ethernetHeaderSrcMac = tsRawData.substring(60, 72); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - ctx.ts.ethernetHeaderEtherType = tsRawData.substring(72, 76); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.IPvData = tsRawData.substring(76, 116); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.IPvData = tsRawData.substring(76, 156); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.IPVersion = tsRawData.substring(76, 77); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.IPVersion = tsRawData.substring(76, 77); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String IPv4SrcIPHexString = tsRawData.substring(100, 108); - String IPv4SrcIP = ""; - for(int i = 0; i < IPv4SrcIPHexString.length(); i = i + 2) { - IPv4SrcIP = IPv4SrcIP + Integer.valueOf(IPv4SrcIPHexString.substring(i, i+2), 16) + "."; - } - IPv4SrcIP = IPv4SrcIP.substring(0, IPv4SrcIP.length()-1); - ctx.ts.IPv4SrcIP = IPv4SrcIP; - } - if( etherType == '86dd' && ipVersion == '6') - { - String IPv6SrcIPHexString = tsRawData.substring(92,124); - String IPv6SrcIP = ""; - for(int i = 0; i < IPv6SrcIPHexString.length(); i = i + 4) { - IPv6SrcIP = IPv6SrcIP + IPv6SrcIPHexString.substring(i, i+4) + ":"; - } - IPv6SrcIP = IPv6SrcIP.substring(0, IPv6SrcIP.length()-1); - ctx.ts.IPv6SrcIP = IPv6SrcIP; - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String IPv4DestIPHexString = tsRawData.substring(108, 116); - String IPv4DestIP = ""; - for(int i = 0; i < IPv4DestIPHexString.length(); i = i + 2) { - IPv4DestIP = IPv4DestIP + Integer.valueOf(IPv4DestIPHexString.substring(i, i+2), 16) + "."; - } - IPv4DestIP = IPv4DestIP.substring(0, IPv4DestIP.length()-1); - ctx.ts.IPv4DestIP = IPv4DestIP; - } - if( etherType == '86dd' && ipVersion == '6') - { - String IPv6DestIPHexString = tsRawData.substring(124,156); - String IPv6DestIP = ""; - for(int i = 0; i < IPv6DestIPHexString.length(); i = i + 4) { - IPv6DestIP = IPv6DestIP + IPv6DestIPHexString.substring(i, i+4) + ":"; - } - IPv6DestIP = IPv6DestIP.substring(0, IPv6DestIP.length()-1); - ctx.ts.IPv6DestIP = IPv6DestIP; - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDGPData = tsRawData.substring(116, 132); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDGPData = tsRawData.substring(156, 172); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDGSrcPortStr = tsRawData.substring(116, 120); - BigInteger UDGSrcPortBigInt = new BigInteger(UDGSrcPortStr, 16); - ctx.ts.UDGSrcPort = UDGSrcPortBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDGSrcPortStr = tsRawData.substring(156, 160); - BigInteger UDGSrcPortBigInt = new BigInteger(UDGSrcPortStr, 16); - ctx.ts.UDGSrcPort = UDGSrcPortBigInt.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDGDstPortStr = tsRawData.substring(120, 124); - BigInteger UDGDstPortBigInt = new BigInteger(UDGDstPortStr, 16); - ctx.ts.UDGDstPort = UDGDstPortBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDGDstPortStr = tsRawData.substring(160, 164); - BigInteger UDGDstPortBigInt = new BigInteger(UDGDstPortStr, 16); - ctx.ts.UDGDstPort = UDGDstPortBigInt.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDGLengthStr = tsRawData.substring(124, 128); - BigInteger UDGLengthBigInt = new BigInteger(UDGLengthStr, 16); - ctx.ts.UDGLength = UDGLengthBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDGLengthStr = tsRawData.substring(164, 168); - BigInteger UDGLengthBigInt = new BigInteger(UDGLengthStr, 16); - ctx.ts.UDGLength = UDGLengthBigInt.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDGChecksumStr = tsRawData.substring(128, 132); - BigInteger UDGChecksumBigInt = new BigInteger(UDGChecksumStr, 16); - ctx.ts.UDGChecksum = UDGChecksumBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDGChecksumStr = tsRawData.substring(168, 172); - BigInteger UDGChecksumBigInt = new BigInteger(UDGChecksumStr, 16); - ctx.ts.UDGChecksum = UDGChecksumBigInt.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.InBandNetworkTelemetryData = tsRawData.substring(132, 196); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.InBandNetworkTelemetryData = tsRawData.substring(172, 236); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDPIntShimHeaderData = tsRawData.substring(132, 140); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDPIntShimHeaderData = tsRawData.substring(172, 180); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDPIntShimHeaderType = tsRawData.substring(132, 133); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDPIntShimHeaderType = tsRawData.substring(172, 173); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String udpIntShimNPTStr = tsRawData.substring(133, 134); - BigInteger udpIntShimNPTBigInt = new BigInteger(udpIntShimNPTStr, 16); - BigInteger bitwiseAnd = new BigInteger('C', 16); - BigInteger result = udpIntShimNPTBigInt.and(bitwiseAnd); - ctx.ts.UDPIntShimHeaderNPT = result.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String udpIntShimNPTStr = tsRawData.substring(173, 174); - BigInteger udpIntShimNPTBigInt = new BigInteger(udpIntShimNPTStr, 16); - BigInteger bitwiseAnd = new BigInteger('C', 16); - BigInteger result = udpIntShimNPTBigInt.and(bitwiseAnd); - ctx.ts.UDPIntShimHeaderNPT = result.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDPIntShimHeaderRes1 = tsRawData.substring(133, 134); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDPIntShimHeaderRes1 = tsRawData.substring(174, 175); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String lengthStr = tsRawData.substring(134, 136); - BigInteger lengthStrBigInt = new BigInteger(lengthStr, 16); - ctx.ts.UDPIntShimHeaderLength= lengthStrBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String lengthStr = tsRawData.substring(174, 176); - BigInteger lengthStrBigInt = new BigInteger(lengthStr, 16); - ctx.ts.UDPIntShimHeaderLength = lengthStrBigInt.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDPIntShimHeaderRes2 = tsRawData.substring(136, 138); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDPIntShimHeaderRes2 = tsRawData.substring(176, 178); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String nextProtoStr = tsRawData.substring(138, 140); - BigInteger nextProtoBigInt = new BigInteger(nextProtoStr, 16); - ctx.ts.UDPIntShimHeaderNextProto = nextProtoBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String nextProtoStr = tsRawData.substring(178, 180); - BigInteger nextProtoBigInt = new BigInteger(nextProtoStr, 16); - ctx.ts.UDPIntShimHeaderNextProto = nextProtoBigInt.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.INTMetadataHeaderData = tsRawData.substring(140, 164); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.INTMetadataHeaderData = tsRawData.substring(180, 204); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderVersionStr = tsRawData.substring(140, 141); - BigInteger UDPIntMetaDataHeaderVersionStrBigInt = new BigInteger(UDPIntMetaDataHeaderVersionStr, 16); - ctx.ts.UDPIntMetaDataHeaderVersion = UDPIntMetaDataHeaderVersionStrBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderVersionStr = tsRawData.substring(180, 181); - BigInteger UDPIntMetaDataHeaderVersionStrBigInt = new BigInteger(UDPIntMetaDataHeaderVersionStr, 16); - ctx.ts.UDPIntMetaDataHeaderVersion = UDPIntMetaDataHeaderVersionStrBigInt.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderRes = tsRawData.substring(141, 142); - BigInteger UDPIntMetaDataHeaderResBigInt = new BigInteger(UDPIntMetaDataHeaderRes, 16); - BigInteger bitwiseAnd = new BigInteger('C', 16); - BigInteger result = UDPIntMetaDataHeaderResBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderRes = result.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderRes = tsRawData.substring(181, 182); - BigInteger UDPIntMetaDataHeaderResBigInt = new BigInteger(UDPIntMetaDataHeaderRes, 16); - BigInteger bitwiseAnd = new BigInteger('C', 16); - BigInteger result = UDPIntMetaDataHeaderResBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderRes = result.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderD = tsRawData.substring(141, 142); - BigInteger UDPIntMetaDataHeaderDBigInt = new BigInteger(UDPIntMetaDataHeaderD, 16); - BigInteger bitwiseAnd = new BigInteger('2', 16); - BigInteger result = UDPIntMetaDataHeaderDBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderD = result.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderD = tsRawData.substring(181, 182); - BigInteger UDPIntMetaDataHeaderDBigInt = new BigInteger(UDPIntMetaDataHeaderD, 16); - BigInteger bitwiseAnd = new BigInteger('2', 16); - BigInteger result = UDPIntMetaDataHeaderDBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderD = result.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderE = tsRawData.substring(141, 142); - BigInteger UDPIntMetaDataHeaderEBigInt = new BigInteger(UDPIntMetaDataHeaderE, 16); - BigInteger bitwiseAnd = new BigInteger('1', 16); - BigInteger result = UDPIntMetaDataHeaderEBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderE = result.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderE = tsRawData.substring(181, 182); - BigInteger UDPIntMetaDataHeaderEBigInt = new BigInteger(UDPIntMetaDataHeaderE, 16); - BigInteger bitwiseAnd = new BigInteger('1', 16); - BigInteger result = UDPIntMetaDataHeaderEBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderE = result.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderM = tsRawData.substring(142, 143); - BigInteger UDPIntMetaDataHeaderMBigInt = new BigInteger(UDPIntMetaDataHeaderM, 16); - BigInteger bitwiseAnd = new BigInteger('8', 16); - BigInteger result = UDPIntMetaDataHeaderMBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderM = result.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderM = tsRawData.substring(182, 183); - BigInteger UDPIntMetaDataHeaderMBigInt = new BigInteger(UDPIntMetaDataHeaderM, 16); - BigInteger bitwiseAnd = new BigInteger('8', 16); - BigInteger result = UDPIntMetaDataHeaderMBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderM = result.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderPerhopMetadataLengthStr = tsRawData.substring(144, 146); - BigInteger UDPIntMetaDataHeaderPerhopMetadataLengthBigInt = new BigInteger(UDPIntMetaDataHeaderPerhopMetadataLengthStr, 16); - BigInteger bitwiseAnd = new BigInteger('1F', 16); - BigInteger result = UDPIntMetaDataHeaderPerhopMetadataLengthBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderPerhopMetadataLength = result.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderPerhopMetadataLengthStr = tsRawData.substring(184, 186); - BigInteger UDPIntMetaDataHeaderPerhopMetadataLengthBigInt = new BigInteger(UDPIntMetaDataHeaderPerhopMetadataLengthStr, 16); - BigInteger bitwiseAnd = new BigInteger('1F', 16); - BigInteger result = UDPIntMetaDataHeaderPerhopMetadataLengthBigInt.and(bitwiseAnd); - ctx.ts.UDPIntMetaDataHeaderPerhopMetadataLength = result.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderRemainingHopCntStr = tsRawData.substring(146, 148); - BigInteger UDPIntMetaDataHeaderRemainingHopCntBigInt = new BigInteger(UDPIntMetaDataHeaderRemainingHopCntStr, 16); - ctx.ts.UDPIntMetaDataHeaderRemainingHopCnt = UDPIntMetaDataHeaderRemainingHopCntBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderRemainingHopCntStr = tsRawData.substring(186, 188); - BigInteger UDPIntMetaDataHeaderRemainingHopCntBigInt = new BigInteger(UDPIntMetaDataHeaderRemainingHopCntStr, 16); - ctx.ts.UDPIntMetaDataHeaderRemainingHopCnt = UDPIntMetaDataHeaderRemainingHopCntBigInt.intValue(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderInstructionBitmapStr = tsRawData.substring(148, 152); - BigInteger UDPIntMetaDataHeaderInstructionBitmapBigInt = new BigInteger(UDPIntMetaDataHeaderInstructionBitmapStr, 16); - ctx.ts.UDPIntMetaDataHeaderInstructionBitmap = UDPIntMetaDataHeaderInstructionBitmapBigInt.toString(16); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderInstructionBitmapStr = tsRawData.substring(188, 192); - BigInteger UDPIntMetaDataHeaderInstructionBitmapBigInt = new BigInteger(UDPIntMetaDataHeaderInstructionBitmapStr, 16); - ctx.ts.UDPIntMetaDataHeaderInstructionBitmap = UDPIntMetaDataHeaderInstructionBitmapBigInt.toString(16); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderDomainSpecificIDStr = tsRawData.substring(152, 156); - BigInteger UDPIntMetaDataHeaderDomainSpecificIDBigInt = new BigInteger(UDPIntMetaDataHeaderDomainSpecificIDStr, 16); - ctx.ts.UDPIntMetaDataHeaderDomainSpecificID = UDPIntMetaDataHeaderDomainSpecificIDBigInt.toString(16); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderDomainSpecificIDStr = tsRawData.substring(192, 196); - BigInteger UDPIntMetaDataHeaderDomainSpecificIDBigInt = new BigInteger(UDPIntMetaDataHeaderDomainSpecificIDStr, 16); - ctx.ts.UDPIntMetaDataHeaderDomainSpecificID = UDPIntMetaDataHeaderDomainSpecificIDBigInt.toString(16); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderDSInstructionStr = tsRawData.substring(156, 160); - BigInteger UDPIntMetaDataHeaderDSInstructionBigInt = new BigInteger(UDPIntMetaDataHeaderDSInstructionStr, 16); - ctx.ts.UDPIntMetaDataHeaderDSInstruction = UDPIntMetaDataHeaderDSInstructionBigInt.toString(16); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderDSInstructionStr = tsRawData.substring(196, 200); - BigInteger UDPIntMetaDataHeaderDSInstructionBigInt = new BigInteger(UDPIntMetaDataHeaderDSInstructionStr, 16); - ctx.ts.UDPIntMetaDataHeaderDSInstruction = UDPIntMetaDataHeaderDSInstructionBigInt.toString(16); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String UDPIntMetaDataHeaderDSFlagsStr = tsRawData.substring(160, 164); - BigInteger UDPIntMetaDataHeaderDSFlagsBigInt = new BigInteger(UDPIntMetaDataHeaderDSFlagsStr, 16); - ctx.ts.UDPIntMetaDataHeaderDSFlags = UDPIntMetaDataHeaderDSFlagsBigInt.toString(16); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDPIntMetaDataHeaderDSFlagsStr = tsRawData.substring(200, 204); - BigInteger UDPIntMetaDataHeaderDSFlagsBigInt = new BigInteger(UDPIntMetaDataHeaderDSFlagsStr, 16); - ctx.ts.UDPIntMetaDataHeaderDSFlags = UDPIntMetaDataHeaderDSFlagsBigInt.toString(16); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.INTMetadataStackData = tsRawData.substring(164, 196); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.INTMetadataStackData = tsRawData.substring(204, 236); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.INTMetadataStackSwitchID = tsRawData.substring(164, 172); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.INTMetadataStackSwitchID = tsRawData.substring(204, 212); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.INTMetadataSourceMetadata = tsRawData.substring(172, 196); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.INTMetadataSourceMetadata = tsRawData.substring(212, 236); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.INTMetadataSourceMetadataSwitchId = tsRawData.substring(172, 180); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.INTMetadataSourceMetadataSwitchId = tsRawData.substring(212, 220); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - String rawOriginatingMac = tsRawData.substring(180, 192); - StringBuilder formattedOriginatingMac = new StringBuilder(); - for(int i = 0; i < rawOriginatingMac.length();) - { - formattedOriginatingMac.append(rawOriginatingMac.substring(i++, ++i)); - if(i < rawOriginatingMac.length()) formattedOriginatingMac.append(":"); - } - ctx.ts.INTMetadataSourceMetadataOriginatingMac = formattedOriginatingMac.toString(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String rawOriginatingMac = tsRawData.substring(220, 232); - StringBuilder formattedOriginatingMac = new StringBuilder(); - for(int i = 0; i < rawOriginatingMac.length();) - { - formattedOriginatingMac.append(rawOriginatingMac.substring(i++, ++i)); - if(i < rawOriginatingMac.length()) formattedOriginatingMac.append(":"); - } - ctx.ts.INTMetadataSourceMetadataOriginatingMac = formattedOriginatingMac.toString(); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.INTMetadataSourceMetadataReserved = tsRawData.substring(192, 196); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.INTMetadataSourceMetadataReserved = tsRawData.substring(232, 236); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if(ctx.ts.isUDPv4 == true || ctx.ts.isUDPv6 == true) - { - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDP2Data = tsRawData.substring(196, 212); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDP2Data = tsRawData.substring(236, 252); - } - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - String tcpNextProtoStr = '6'; - int tcpNextProtoInt = Integer.parseInt(tcpNextProtoStr); - if(ctx.ts.isTCPv4 == true || ctx.ts.isTCPv6 == true) - { - if( etherType == '0800' && ipVersion == '4' && ctx.ts.nextProto == tcpNextProtoInt ) - { - ctx.ts.TCPData = tsRawData.substring(196, 236); - } - if(etherType == '86dd' && ipVersion == '6' && ctx.ts.nextProto == tcpNextProtoInt ) - { - ctx.ts.TCPData = tsRawData.substring(236, 276); - } - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( ctx.ts.isUDPv4 == true || ctx.ts.isUDPv6 == true ) - { - if( etherType == '0800' && ipVersion == '4') - { - String UDP2SrcPortStr = tsRawData.substring(196, 200); - BigInteger UDP2SrcPortBigInt = new BigInteger(UDP2SrcPortStr,16); - ctx.ts.UDP2SrcPort = UDP2SrcPortBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDP2SrcPortStr = tsRawData.substring(236, 240); - BigInteger UDP2SrcPortBigInt = new BigInteger(UDP2SrcPortStr,16); - ctx.ts.UDP2SrcPort = UDP2SrcPortBigInt.intValue(); - } - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - String tcpNextProtoStr = '6'; - if( ctx.ts.isTCPv4 == true || ctx.ts.isTCPv6 == true ) - { - int tcpNextProtoInt = Integer.parseInt(tcpNextProtoStr); - if( etherType == '0800' && ipVersion == '4' ) - { - String TCPSrcPortStr = tsRawData.substring(196, 200); - BigInteger TCPSrcPortBigInt = new BigInteger(TCPSrcPortStr,16); - ctx.ts.TCPSrcPort = TCPSrcPortBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String TCPSrcPortStr = tsRawData.substring(236, 240); - BigInteger TCPSrcPortBigInt = new BigInteger(TCPSrcPortStr,16); - ctx.ts.TCPSrcPort = TCPSrcPortBigInt.intValue(); - } - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( ctx.ts.isUDPv4 == true || ctx.ts.isUDPv6 == true ) - { - if( etherType == '0800' && ipVersion == '4') - { - String UDP2DstPortStr = tsRawData.substring(200, 204); - BigInteger UDP2DstPortBigInt = new BigInteger(UDP2DstPortStr,16); - ctx.ts.UDP2DstPort = UDP2DstPortBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6') - { - String UDP2DstPortStr = tsRawData.substring(240, 244); - BigInteger UDP2DstPortBigInt = new BigInteger(UDP2DstPortStr,16); - ctx.ts.UDP2DstPort = UDP2DstPortBigInt.intValue(); - } - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - String tcpNextProtoStr = '6'; - if( ctx.ts.isTCPv4 == true || ctx.ts.isTCPv6 == true ) - { - int tcpNextProtoInt = Integer.parseInt(tcpNextProtoStr); - if( etherType == '0800' && ipVersion == '4' && ctx.ts.nextProto == tcpNextProtoInt ) - { - String TCPDstPortStr = tsRawData.substring(200, 204); - BigInteger TCPDstPortBigInt = new BigInteger(TCPDstPortStr,16); - ctx.ts.TCPDestPort = TCPDstPortBigInt.intValue(); - } - if( etherType == '86dd' && ipVersion == '6' && ctx.ts.nextProto == tcpNextProtoInt ) - { - String TCPDstPortStr = tsRawData.substring(240, 244); - BigInteger TCPDstPortBigInt = new BigInteger(TCPDstPortStr,16); - ctx.ts.TCPDestPort = TCPDstPortBigInt.intValue(); - } - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDP2Length = tsRawData.substring(204, 208); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDP2Length = tsRawData.substring(244, 248); - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == true) - { - String dropTelemetryReport = ctx.ts.dropTelemetryReport; - String telemetryReportVarOpBsmdStr = dropTelemetryReport.substring(40,48); - BigInteger telemetryReportVarOpBsmdBigInt = new BigInteger(telemetryReportVarOpBsmdStr, 16); - ctx.ts.telemetryReportVarOpBsmd = telemetryReportVarOpBsmdBigInt.intValue(); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == true) - { - String dropTelemetryReport = ctx.ts.dropTelemetryReport; - String telemetryReportVarOptTimeStampStr = dropTelemetryReport.substring(48,56); - BigInteger telemetryReportVarOptTimeStampBigInt = new BigInteger(telemetryReportVarOptTimeStampStr, 16); - ctx.ts.telemetryReportVarOptTimeStamp = telemetryReportVarOptTimeStampBigInt.intValue(); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == true) - { - String dropTelemetryReport = ctx.ts.dropTelemetryReport; - String telemetryReportVarOptDropCountStr = dropTelemetryReport.substring(56,64); - BigInteger telemetryReportVarOptDropCountBigInt = new BigInteger(telemetryReportVarOptDropCountStr, 16); - ctx.ts.telemetryReportVarOptDropCount = telemetryReportVarOptDropCountBigInt.intValue(); - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == true) - { - String dropTelemetryReport = ctx.ts.dropTelemetryReport; - String telemetryReportVarOptHashStr = dropTelemetryReport.substring(64,96); - BigInteger telemetryReportVarOptHashBigInt = new BigInteger(telemetryReportVarOptHashStr, 16); - ctx.ts.telemetryReportVarOptHash = telemetryReportVarOptHashBigInt; - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - if(ctx.ts.isUDPv4 == true || ctx.ts.isUDPv6 == true) - { - String macAddress = ctx.ts.INTMetadataSourceMetadataOriginatingMac; - String destinationPort = ctx.ts.UDP2DstPort.toString(); - String combinedKeys = ''; - if(ctx.ts.isIPv4 == true && ctx.ts.isUDPv4 == true) - { - String IPvDestIP = ctx.ts.IPv4DestIP; - combinedKeys = combinedKeys.concat(macAddress).concat('|').concat(destinationPort).concat('|').concat(IPvDestIP).concat('|').concat('::'); - ctx.ts.dropHashInput = combinedKeys; - } - if(ctx.ts.isIPv6 == true && ctx.ts.isUDPv6 == true) - { - String IPvDestIP = ctx.ts.IPv6DestIP; - combinedKeys = combinedKeys.concat(macAddress).concat('|').concat(destinationPort).concat('|').concat(IPvDestIP).concat('|').concat('::'); - ctx.ts.dropHashInput = combinedKeys; - } - } - if(ctx.ts.isTCPv4 == true || ctx.ts.isTCPv6 == true) - { - String macAddress = ctx.ts.INTMetadataSourceMetadataOriginatingMac; - String destinationPort = ctx.ts.TCPDestPort.toString(); - String combinedKeys = ''; - if(ctx.ts.isIPv4 == true && ctx.ts.isTCPv4 == true) - { - String IPvDestIP = ctx.ts.IPv4DestIP; - combinedKeys = combinedKeys.concat(macAddress).concat('|').concat(destinationPort).concat('|').concat(IPvDestIP).concat('|').concat('::'); - ctx.ts.dropHashInput = combinedKeys; - } - if(ctx.ts.isIPv6 == true && ctx.ts.isTCPv6 == true) - { - String IPvDestIP = ctx.ts.IPv6DestIP; - combinedKeys = combinedKeys.concat(macAddress).concat('|').concat(destinationPort).concat('|').concat(IPvDestIP).concat('|').concat('::'); - ctx.ts.dropHashInput = combinedKeys; - } - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "ignore_failure" : false, - "id" : "calculateKeyHash" - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - String completeSHA256Hash = ctx.ts.sha256Hash; - String subsetSHA256HashStr= completeSHA256Hash.substring(0,16); - BigInteger hashBigInt = new BigInteger(subsetSHA256HashStr, 16); - ctx.ts.dropReportHash = hashBigInt; - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if(ctx.ts.isUDPv4 == true || ctx.ts.isUDPv6 == true) - { - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.UDP2Checksum = tsRawData.substring(208, 212); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.UDP2Checksum = tsRawData.substring(248, 252); - } - } - } - """, - "ignore_failure" : true - } - }, - { - "script" : { - "lang" : "painless", - "source" : """ - if(ctx.ts.isDropReport == false) - { - String tsRawData = ctx.layers.data.data_data_data.replace(':',''); - String etherType = tsRawData.substring(72, 76); - String ipVersion = tsRawData.substring(76, 77); - if (ctx.ts.isUDPv4 == true || ctx.ts.isUDPv6 == true) - { - if( etherType == '0800' && ipVersion == '4') - { - ctx.ts.dataII = tsRawData.substring(212, 252); - } - if( etherType == '86dd' && ipVersion == '6') - { - ctx.ts.dataII = tsRawData.substring(252, 292); - } - } - if(ctx.ts.isTCPv4 == true || ctx.ts.isTCPv6 == true) - { - String tcpNextProtoStr = '6'; - int tcpNextProtoInt = Integer.parseInt(tcpNextProtoStr); - if( etherType == '0800' && ipVersion == '4' ) - { - ctx.ts.dataII = tsRawData.substring(236, 288); - } - if(etherType == '86dd' && ipVersion == '6' ) - { - ctx.ts.dataII = tsRawData.substring(252, 304); - } - } - } - """, - "ignore_failure" : true - } - } - ] - } diff --git a/playbooks/hcp/templates/udp_and_tcp_monitor_def.json b/playbooks/hcp/templates/udp_and_tcp_monitor_def.json deleted file mode 100644 index 73774473..00000000 --- a/playbooks/hcp/templates/udp_and_tcp_monitor_def.json +++ /dev/null @@ -1,347 +0,0 @@ -{ - "type": "monitor", - "name": "DDOS-UDP-TCP-Monitor", - "enabled": true, - "schedule": { - "period": { - "interval": 1, - "unit": "MINUTES" - } - }, - "inputs": [{ - "search": { - "indices": ["packets-*"], - "query": { - "size": 0, - "query": { - "bool": { - "filter": [ - { - "range": { - "timestamp": { - "from": "{{period_end}}||-1m", - "to": "{{period_end}}", - "include_lower": true, - "include_upper": true, - "format": "epoch_millis", - "boost": 1 - } - } - } - ], - "adjust_pure_negative": true, - "boost": 1 - } - }, - "_source": { - "includes": [ - "ts.UDP2DstPort", - "ts.TCPDestPort", - "ts.IPvDestAddr", - "ts.INTMetadataSourceMetadataOriginatingMac", - "ts.IPv4SrcIP", - "ts.IPv4DestIP", - "ts.IPv6SrcIP", - "ts.IPv6DestIP", - "ts.isTCPv4", - "ts.isTCPv6", - "ts.isUDPv4", - "ts.isUDPv6" - ], - "excludes": [] - }, - "aggregations": { - "IPvDestAddrValueCount": { - "value_count": { - "field": "ts.IPvDestAddr" - } - }, - "UDP2DstPortValue": { - "terms": { - "field": "ts.UDP2DstPort", - "size": 10, - "min_doc_count": 1, - "shard_min_doc_count": 0, - "show_term_doc_count_error": false, - "order": [ - { - "_count": "desc" - }, - { - "_key": "asc" - } - ] - } - }, - "TCPDstPortValue": { - "terms": { - "field": "ts.TCPDestPort", - "size": 10, - "min_doc_count": 1, - "shard_min_doc_count": 0, - "show_term_doc_count_error": false, - "order": [ - { - "_count": "desc" - }, - { - "_key": "asc" - } - ] - } - }, - "INTMetadataSourceMetadataOriginatingMacValueCount": { - "value_count": { - "field": "ts.INTMetadataSourceMetadataOriginatingMac" - } - }, - "IPv4DestIP": { - "terms": { - "field": "ts.IPv4DestIP", - "size": 10, - "min_doc_count": 1, - "shard_min_doc_count": 0, - "show_term_doc_count_error": false, - "order": [ - { - "_count": "desc" - }, - { - "_key": "asc" - } - ] - } - }, - "UDP2DstPortValueCount": { - "value_count": { - "field": "ts.UDP2DstPort" - } - }, - "TCPDstPortValueCount": { - "value_count": { - "field": "ts.TCPDestPort" - } - }, - "IPv4SrcIP": { - "terms": { - "field": "ts.IPv4SrcIP", - "size": 10, - "min_doc_count": 1, - "shard_min_doc_count": 0, - "show_term_doc_count_error": false, - "order": [ - { - "_count": "desc" - }, - { - "_key": "asc" - } - ] - } - }, - "IPv6SrcIP": { - "terms": { - "field": "ts.IPv6SrcIP", - "size": 10, - "min_doc_count": 1, - "shard_min_doc_count": 0, - "show_term_doc_count_error": false, - "order": [ - { - "_count": "desc" - }, - { - "_key": "asc" - } - ] - } - }, - "IPv6DestIP": { - "terms": { - "field": "ts.IPv6DestIP", - "size": 10, - "min_doc_count": 1, - "shard_min_doc_count": 0, - "show_term_doc_count_error": false, - "order": [ - { - "_count": "desc" - }, - { - "_key": "asc" - } - ] - } - }, - "OriginatingMacAddrUDPv4": { - "terms": { - "field": "ts.INTMetadataSourceMetadataOriginatingMac", - "size": 10, - "min_doc_count": 1, - "shard_min_doc_count": 0, - "show_term_doc_count_error": false, - "order": [ - { - "_count": "desc" - }, - { - "_key": "asc" - } - ] - } - }, - "isTCPv4Count": { - "terms": { - "field": "ts.isTCPv4", - "size": 10, - "min_doc_count": 1, - "shard_min_doc_count": 0, - "show_term_doc_count_error": false, - "order": [ - - { - "_key": "desc" - } - ] - } - }, - "isTCPv6Count": { - "terms": { - "field": "ts.isTCPv6", - "size": 10, - "min_doc_count": 1, - "shard_min_doc_count": 0, - "show_term_doc_count_error": false, - "order": [ - - { - "_key": "desc" - } - ] - } - }, - "isUDPv4Count": { - "terms": { - "field": "ts.isUDPv4", - "size": 10, - "min_doc_count": 1, - "shard_min_doc_count": 0, - "show_term_doc_count_error": false, - "order": [ - - { - "_key": "desc" - } - ] - } - }, - "isUDPv6Count": { - "terms": { - "field": "ts.isUDPv6", - "size": 10, - "min_doc_count": 1, - "shard_min_doc_count": 0, - "show_term_doc_count_error": false, - "order": [ - - { - "_key": "desc" - } - ] - } - } - } - } - } - }], - "triggers": [ - { - "name": "DDOS-Trigger-TCPv4", - "severity": "1", - "condition": { - "script": { - "source": "if(ctx.results[0].hits.total.value > 0){ctx.results[0].aggregations.TCPDstPortValueCount.value >= 100 && ctx.results[0].aggregations.isTCPv4Count.buckets.length > 0 && ctx.results[0].aggregations.isTCPv4Count.buckets.0.key =='true' && ctx.results[0].aggregations.isTCPv4Count.buckets.0.doc_count >= 100}", - "lang": "painless" - } - }, - "actions": [{ - "name": "Sdn-WebHook-Action", - "destination_id": "sdnWebhookResponseIdTCPv4", - "message_template": { - "source":"{\"event_action\":\"trigger\", \"dst_ip\":\"{{ctx.results.0.aggregations.IPv4DestIP.buckets.0.key}}\",\"dst_port\":\"{{ctx.results.0.aggregations.TCPDstPortValue.buckets.0.key}}\",\"src_ip\":\"{{ctx.results.0.aggregations.IPv4SrcIP.buckets.0.key}}\",\"src_mac\":\"{{ctx.results.0.aggregations.OriginatingMacAddrUDPv4.buckets.0.key}}\"}", - "lang": "mustache", - "options": { - "content_type": "application/json" - } - }, - "throttle_enabled": false - }] - }, - { - "name": "DDOS-Trigger-TCPv6", - "severity": "1", - "condition": { - "script": { - "source": "if(ctx.results[0].hits.total.value > 0){ctx.results[0].aggregations.TCPDstPortValueCount.value >= 100 && ctx.results[0].aggregations.isTCPv6Count.buckets.length > 0 && ctx.results[0].aggregations.isTCPv6Count.buckets.0.key =='true' && ctx.results[0].aggregations.isTCPv6Count.buckets.0.doc_count >= 100}", - "lang": "painless" - } - }, - "actions": [{ - "name": "Sdn-WebHook-Action", - "destination_id": "sdnWebhookResponseIdTCPv6", - "message_template": { - "source":"{\"event_action\":\"trigger\", \"dst_ip\":\"{{ctx.results.0.aggregations.IPv6DestIP.buckets.0.key}}\",\"dst_port\":\"{{ctx.results.0.aggregations.TCPDstPortValue.buckets.0.key}}\",\"src_ip\":\"{{ctx.results.0.aggregations.IPv6SrcIP.buckets.0.key}}\",\"src_mac\":\"{{ctx.results.0.aggregations.OriginatingMacAddrUDPv4.buckets.0.key}}\"}", - "lang": "mustache", - "options": { - "content_type": "application/json" - } - }, - "throttle_enabled": false - }] - }, - { - "name": "DDOS-Trigger-UDPv4", - "severity": "1", - "condition": { - "script": { - "source": "if(ctx.results[0].hits.total.value > 0){ctx.results[0].aggregations.UDP2DstPortValueCount.value >= 100 && ctx.results[0].aggregations.isUDPv4Count.buckets.length > 0 && ctx.results[0].aggregations.isUDPv4Count.buckets.0.key =='true' && ctx.results[0].aggregations.isUDPv4Count.buckets.0.doc_count >= 100}", - "lang": "painless" - } - }, - "actions": [{ - "name": "Sdn-WebHook-Action", - "destination_id": "sdnWebhookResponseIdUDPv4", - "message_template": { - "source":"{\"event_action\":\"trigger\", \"dst_ip\":\"{{ctx.results.0.aggregations.IPv4DestIP.buckets.0.key}}\",\"dst_port\":\"{{ctx.results.0.aggregations.UDP2DstPortValue.buckets.0.key}}\",\"src_ip\":\"{{ctx.results.0.aggregations.IPv4SrcIP.buckets.0.key}}\",\"src_mac\":\"{{ctx.results.0.aggregations.OriginatingMacAddrUDPv4.buckets.0.key}}\"}", - "lang": "mustache", - "options": { - "content_type": "application/json" - } - }, - "throttle_enabled": false - }] - }, - { - "name": "DDOS-Trigger-UDPv6", - "severity": "1", - "condition": { - "script": { - "source": "if(ctx.results[0].hits.total.value > 0){ctx.results[0].aggregations.UDP2DstPortValueCount.value >= 100 && ctx.results[0].aggregations.isUDPv6Count.buckets.length > 0 && ctx.results[0].aggregations.isUDPv6Count.buckets.0.key =='true' && ctx.results[0].aggregations.isUDPv6Count.buckets.0.doc_count >= 100}", - "lang": "painless" - } - }, - "actions": [{ - "name": "Sdn-WebHook-Action", - "destination_id": "sdnWebhookResponseIdUDPv6", - "message_template": { - "source":"{\"event_action\":\"trigger\", \"dst_ip\":\"{{ctx.results.0.aggregations.IPv6DestIP.buckets.0.key}}\",\"dst_port\":\"{{ctx.results.0.aggregations.UDP2DstPortValue.buckets.0.key}}\",\"src_ip\":\"{{ctx.results.0.aggregations.IPv6SrcIP.buckets.0.key}}\",\"src_mac\":\"{{ctx.results.0.aggregations.OriginatingMacAddrUDPv4.buckets.0.key}}\"}", - "lang": "mustache", - "options": { - "content_type": "application/json" - } - }, - "throttle_enabled": false - }] - } - ] -} diff --git a/playbooks/hcp/toggle_ae_monitors.yml b/playbooks/hcp/toggle_ae_monitors.yml deleted file mode 100644 index 90d9f03d..00000000 --- a/playbooks/hcp/toggle_ae_monitors.yml +++ /dev/null @@ -1,49 +0,0 @@ - -- hosts: ae - gather_facts: no - tasks: - - name: Get monitor ID for DDOS-UDP-TCP monitor to update via monitor name - uri: - url: "http://localhost:9200/_opendistro/_alerting/monitors/_search" - method: GET - return_content: yes - body: "{{ lookup('file','./templates/query_monitor_id.json') }}" - body_format: json - register: get_monitor_id_response - - - name: Show retrieved Monitor definition - debug: - var: get_monitor_id_response.json - - - name: Set fact for source from retrieved Monitor definition - set_fact: - monitor_put_original_body: "{{ get_monitor_id_response.json.hits.hits[0]._source }}" - - - name: Change enabled to passed as argument value. - set_fact: - monitor_updated_def: "{{ monitor_put_original_body|combine({'enabled': false }, recursive=True) }}" - when: not toggle | bool - - - name: Change enabled to passed as argument value. - set_fact: - monitor_updated_def: "{{ monitor_put_original_body|combine({'enabled': true }, recursive=True) }}" - when: toggle | bool - - - name: Debug for monitor_updated_def - debug: - var: monitor_updated_def - - - name: PUT call to update Monitor definition in Elasticsearch - uri: - url: "http://localhost:9200/_opendistro/_alerting/monitors/{{get_monitor_id_response.json.hits.hits[0]._id}}" - method: PUT - return_content: yes - status_code: 200 - body: "{{ monitor_updated_def }}" - body_format: json - register: updatedMonitorDefResponse - - - name: Fetch updated monitor definition. - debug: - msg: " Updated Monitor Definition : {{ updatedMonitorDefResponse }}" - diff --git a/playbooks/mininet/ae_tunnel.yml b/playbooks/mininet/ae_tunnel.yml deleted file mode 100644 index e5f9238b..00000000 --- a/playbooks/mininet/ae_tunnel.yml +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright (c) 2020 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -- hosts: all - gather_facts: no - become: yes - tasks: - - name: Create GRE tunnel - command: > - ip link add {{ tunnel_name }} type gretap - local {{ local_ip }} - remote {{ remote_ip }} - - - name: Activate tunnel - command: > - ip link set {{ tunnel_name }} up diff --git a/playbooks/mininet/compile_p4.yml b/playbooks/mininet/compile_p4.yml deleted file mode 100644 index bf64a6d7..00000000 --- a/playbooks/mininet/compile_p4.yml +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -- hosts: "{{ host_val | default('all') }}" - gather_facts: no - become: yes - - tasks: - - name: Ensure {{ remote_scripts_dir }} directory has been created - file: - path: "{{ remote_scripts_dir }}/p4" - state: directory - - - name: Compile P4 programs {{ p4_progs }} - command: > - p4c-bm2-ss - --p4v 16 - --p4runtime-format text - --p4runtime-file {{ remote_scripts_dir }}/p4/{{ item }}.p4info - -o {{ remote_scripts_dir }}/p4/{{ item }}.json -D BMV2=foo - {{ trans_sec_dir }}/p4/{{ item }}/{{ item }}.p4 - register: cmd_out - changed_when: cmd_out is not failed - loop: "{{ p4_progs }}" diff --git a/playbooks/mininet/generate_inventory.yml b/playbooks/mininet/generate_inventory.yml deleted file mode 100644 index 91cd4bea..00000000 --- a/playbooks/mininet/generate_inventory.yml +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -- hosts: all - gather_facts: no - tasks: - - name: Create ansible inventory file {{ remote_inventory_file }} - file: - path: "{{ remote_inventory_file }}" - state: touch - - - name: Create inventory variables heading at {{ remote_inventory_file }} - lineinfile: - path: "{{ remote_inventory_file }}" - line: "[all:vars]" - state: present - - - name: Add variable entries to ansible inventory at {{ remote_inventory_file }} - lineinfile: - path: "{{ remote_inventory_file }}" - line: "{{ item }}" - state: present - loop: - - "trans_sec_dir={{ src_dir }}" - - "srvc_log_dir={{ remote_srvc_log_dir }}" - - "log_level={{ service_log_level }}" - - "log_dir={{ remote_srvc_log_dir }}" - - "remote_scripts_dir={{ remote_scripts_dir }}" - - "remote_ansible_inventory={{ remote_inventory_file }}" - - "topo_file_loc={{ topo_file }}" - - "mn_host_ip={{ mn_host }}" - - "sdn_ip={{ sdn_host }}" - - "sdn_port={{ sdn_port }}" - - "ae_ip={{ ae_host }}" - - "sdn_dev_intf={{ sdn_dev_intf }}" - - "ae_dev_intf={{ ae_dev_intf }}" - - "ae_monitor_intf={{ ae_monitor_intf }}" - - "sdn_url=http://{{ sdn_host }}:{{ sdn_port }}" - - "clone_egress_port={{ clone_egress_port }}" - - "scenario_name={{ scenario_name }}" diff --git a/playbooks/mininet/generate_topology.yml b/playbooks/mininet/generate_topology.yml deleted file mode 100644 index a248fad0..00000000 --- a/playbooks/mininet/generate_topology.yml +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -- hosts: "{{ host_val | default('all') }}" - become: yes - - gather_facts: yes - - vars: - switch_mac: "{{ ansible_facts['eth0']['macaddress'] }}" - - tasks: - - name: Ensure {{ remote_scripts_dir }} scripts directory has been created - file: - path: "{{ remote_scripts_dir }}" - state: directory - - - name: Apply topology configuration template {{ topology_template }} to {{ topo_file_loc }} - become: yes - template: - src: "{{ topology_template }}" - dest: "{{ topo_file_loc }}" diff --git a/playbooks/mininet/local_inventory.yml b/playbooks/mininet/local_inventory.yml deleted file mode 100644 index 65d7caa3..00000000 --- a/playbooks/mininet/local_inventory.yml +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -- hosts: localhost - gather_facts: no - tasks: - - name: copy variable configuration file - vars: - public_ip: "{{ public_ip }}" - template: - src: templates/local_inventory.ini.j2 - dest: "{{ local_inventory }}" - delegate_to: localhost diff --git a/playbooks/mininet/set_ip-full.yml b/playbooks/mininet/set_ip-full.yml deleted file mode 100644 index 33ce05dd..00000000 --- a/playbooks/mininet/set_ip-full.yml +++ /dev/null @@ -1,93 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -- name: Set node facts - set_fact: - north_node: "{{ item.north_node | default(None) }}" - south_node: "{{ item.south_node | default(None) }}" - south_facing_port: "{{ item.south_facing_port | default(None) }}" - north_facing_port: "{{ item.north_facing_port | default(None) }}" - host: "{{ topo_dict.hosts.get(item.south_node) }}" - south_facing_ip: "{{ item.south_facing_ip | default(None) }}" - south_switch: "{{ topo_dict.switches.get(item.south_node) }}" - north_switch: "{{ topo_dict.switches.get(item.north_node) }}" - intf_name: None - -- name: Find north node if south didn't exist - set_fact: - host: "{{ topo_dict.hosts.get(item.north_node) }}" - when: not host - -- name: Determine interface name for south_facing_port - set_fact: - intf_name: "{{ north_node | default('') }}-eth{{ south_facing_port | default('') }}" - when: south_facing_port and not north_facing_port - -- name: Determine interface name for south_facing_port - set_fact: - intf_name: "{{ south_node | default('') }}-eth{{ north_facing_port | default('') }}" - when: north_facing_port and not south_facing_port - -- name: Determine interface name for gateway north-facing port - set_fact: - intf_name: "{{ south_node | default('') }}-eth{{ north_facing_port | default('') }}" - when: north_facing_port and south_facing_port and south_switch - -- name: Set public IP on the gateway - block: - - name: Set public IP for intf {{ intf_name }} - become: yes - command: "ip addr add {{ south_switch.public_ip }}/24 dev {{ intf_name }}" - when: south_switch.public_ip is defined and north_switch - -- name: Setup local IPs on the gateway - block: - - name: Set local IP for intf {{ intf_name }} - become: yes - command: "ip addr add {{ host.switch_ip }}/30 dev {{ intf_name }}" - when: host and north_switch.public_ip is defined - -- name: Setup local IP on the core - block: - - name: Set local IP for core intf {{ intf_name }} - become: yes - command: "ip addr add {{ south_switch.public_ip }}/24 dev {{ intf_name }}" - when: host and south_switch.public_ip is defined - -- name: Install ifmetric - become: yes - apt: - update_cache: yes - name: - - ifmetric - register: apt_rc - retries: 3 - delay: 10 - until: apt_rc is not failed - -- name: Set route priority - become: yes - command: "ifmetric {{ intf_name }} 50" - register: cmd_out - changed_when: cmd_out is not failed - retries: 3 - delay: 5 - -- name: Set route priority for core-inet link - become: yes - command: "ifmetric {{ intf_name }} 25" - register: cmd_out - changed_when: cmd_out is not failed - retries: 3 - delay: 5 - when: host and south_switch.public_ip is defined diff --git a/playbooks/mininet/set_ip-lab_trial.yml b/playbooks/mininet/set_ip-lab_trial.yml deleted file mode 100644 index 67cd48b9..00000000 --- a/playbooks/mininet/set_ip-lab_trial.yml +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -- name: Set node facts - set_fact: - north_node: "{{ item.north_node | default(None) }}" - south_node: "{{ item.south_node | default(None) }}" - south_facing_port: "{{ item.south_facing_port | default(None) }}" - north_facing_port: "{{ item.north_facing_port | default(None) }}" - host: "{{ topo_dict.hosts.get(item.south_node) }}" - south_switch: "{{ topo_dict.switches.get(item.south_node) }}" - north_switch: "{{ topo_dict.switches.get(item.north_node) }}" - north_facing_ip: "{{ item.north_facing_ip | default(None) }}" - south_facing_ip: "{{ item.south_facing_ip | default(None) }}" - intf_name: None - -- name: Find north node if south didn't exist - set_fact: - host: "{{ topo_dict.hosts.get(item.north_node) }}" - when: not host - -- name: Determine interface name for south_facing_port - set_fact: - intf_name: "{{ north_node | default('') }}-eth{{ south_facing_port | default('') }}" - when: south_facing_port and not north_facing_port - -- name: Determine interface name for north_facing_port - set_fact: - intf_name: "{{ south_node | default('') }}-eth{{ north_facing_port | default('') }}" - when: north_facing_port and not south_facing_port - -- name: Set north-facing IP on the core - {{ intf_name }} - become: yes - command: "ip addr add {{ north_facing_ip }}/30 dev {{ intf_name }}" - when: host and south_switch and north_facing_ip is defined - -- name: Setup south-facing IP on the aggregate - {{ intf_name }} - become: yes - command: "ip addr add {{ south_facing_ip }}/30 dev {{ intf_name }}" - when: host and north_switch and south_facing_ip is defined diff --git a/playbooks/mininet/set_ip-single-switch.yml b/playbooks/mininet/set_ip-single-switch.yml deleted file mode 100644 index 0fa8244e..00000000 --- a/playbooks/mininet/set_ip-single-switch.yml +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -- name: Set node facts - set_fact: - north_node: "{{ item.north_node | default(None) }}" - south_node: "{{ item.south_node | default(None) }}" - south_facing_port: "{{ item.south_facing_port | default(None) }}" - north_facing_port: "{{ item.north_facing_port | default(None) }}" - host: "{{ topo_dict.hosts.get(item.south_node) }}" - intf_name: None - -- name: Find north node if south didn't exist - set_fact: - host: "{{ topo_dict.hosts.get(item.north_node) }}" - when: not host - -- name: Determine interface name for south_facing_port - set_fact: - intf_name: "{{ north_node | default('') }}-eth{{ south_facing_port | default('') }}" - when: south_facing_port and not north_facing_port - -- name: Determine interface name for south_facing_port - set_fact: - intf_name: "{{ south_node | default('') }}-eth{{ north_facing_port | default('') }}" - when: north_facing_port and not south_facing_port - -- name: Set IP for intf {{ intf_name }} - become: yes - command: "ip addr add {{ host.switch_ip }}/30 dev {{ intf_name }}" - when: host diff --git a/playbooks/mininet/setup-full.yml b/playbooks/mininet/setup-full.yml deleted file mode 100644 index c93c207d..00000000 --- a/playbooks/mininet/setup-full.yml +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -- import_playbook: start_mininet.yml - vars: - host_val: localhost - service_name: tps-mininet - srvc_desc: 'Mininet for TPS' - local_srvc_script_tmplt_file: ../general/templates/mininet_service.sh.j2 - srvc_start_pause_time: 20 - port_to_wait: 50051 - p4_progs: - - core - - aggregate - - gateway - set_ip_file: set_ip-full.yml - topology_template: templates/topology_template-full.yml.j2 - -- import_playbook: ../general/start_service.yml - vars: - host_val: localhost - service_name: tps-sdn - srvc_desc: 'SDN' - local_srvc_script_tmplt_file: ../general/templates/sdn_controller.sh.j2 - srvc_start_pause_time: 15 - port_to_wait: "{{ sdn_port }}" - p4_platform: bmv2 - load_p4: True - -- import_playbook: ../general/start_service.yml - vars: - host_val: localhost - service_name: tps-ae - srvc_desc: 'TPS-AE' - srvc_type: 'SIMPLE' - local_srvc_script_tmplt_file: ../general/templates/ae_service.sh.j2 - monitor_intf: "{{ ae_monitor_intf }}" diff --git a/playbooks/mininet/setup-lab_trial.yml b/playbooks/mininet/setup-lab_trial.yml deleted file mode 100644 index 5986347a..00000000 --- a/playbooks/mininet/setup-lab_trial.yml +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -- import_playbook: start_mininet.yml - vars: - host_val: localhost - service_name: tps-mininet - srvc_desc: 'Mininet for {{ scenario_name }}.p4' - local_srvc_script_tmplt_file: ../general/templates/mininet_service.sh.j2 - srvc_start_pause_time: 20 - port_to_wait: 50051 - p4_progs: - - core - - aggregate - set_ip_file: set_ip-lab_trial.yml - topology_template: templates/topology_template-lab_trial.yml.j2 - load_p4: False - -- import_playbook: ../general/start_service.yml - vars: - host_val: localhost - service_name: tps-sdn - srvc_desc: 'SDN' - local_srvc_script_tmplt_file: ../general/templates/sdn_controller.sh.j2 - srvc_start_pause_time: 15 - port_to_wait: "{{ sdn_port }}" - p4_platform: bmv2 - load_p4: True diff --git a/playbooks/mininet/setup-single_switch.yml b/playbooks/mininet/setup-single_switch.yml deleted file mode 100644 index 4435df3b..00000000 --- a/playbooks/mininet/setup-single_switch.yml +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -- import_playbook: start_mininet.yml - vars: - host_val: localhost - service_name: tps-mininet - srvc_desc: 'Mininet for {{ scenario_name }}.p4' - local_srvc_script_tmplt_file: ../general/templates/mininet_service.sh.j2 - srvc_start_pause_time: 20 - port_to_wait: 50051 - p4_progs: - - "{{ scenario_name }}" - set_ip_file: set_ip-single-switch.yml - topology_template: "templates/topology_template-single_switch.yml.j2" - -- import_playbook: ../general/start_service.yml - vars: - host_val: localhost - service_name: tps-sdn - srvc_desc: 'SDN' - local_srvc_script_tmplt_file: ../general/templates/sdn_controller.sh.j2 - srvc_start_pause_time: 15 - port_to_wait: "{{ sdn_port }}" - p4_platform: bmv2 - load_p4: True diff --git a/playbooks/mininet/setup_ae2mini_tunnel.yml b/playbooks/mininet/setup_ae2mini_tunnel.yml deleted file mode 100644 index 0c5dcece..00000000 --- a/playbooks/mininet/setup_ae2mini_tunnel.yml +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright (c) 2020 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -- import_playbook: ae_tunnel.yml diff --git a/playbooks/mininet/setup_mini2ae_tunnel.yml b/playbooks/mininet/setup_mini2ae_tunnel.yml deleted file mode 100644 index 92286085..00000000 --- a/playbooks/mininet/setup_mini2ae_tunnel.yml +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright (c) 2020 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -- import_playbook: ae_tunnel.yml - -- hosts: all - gather_facts: no - become: yes - tasks: - - name: Install daemonlogger - apt: - name: daemonlogger - -- import_playbook: ../general/start_service.yml - vars: - host_val: all - service_name: tps-daemon-logger - srvc_desc: 'TPS-CLONE-INTF-MIRROR' - local_srvc_script_tmplt_file: ../general/templates/daemonlogger_service.sh.j2 diff --git a/playbooks/mininet/start_mininet.yml b/playbooks/mininet/start_mininet.yml deleted file mode 100644 index 8d0918cc..00000000 --- a/playbooks/mininet/start_mininet.yml +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -- import_playbook: generate_topology.yml - vars: - host_val: localhost - topology_template: "{{ topology_template }}" - -- import_playbook: compile_p4.yml - vars: - host_val: localhost - p4_progs: "{{ p4_progs }}" - -- import_playbook: ../general/start_service.yml - vars: - host_val: localhost - -- hosts: localhost - gather_facts: no - become: yes - - vars: - topo_file_dest: "{{ topo_file_remote_loc | default('/tmp/mininet-topology.yaml') }}" - - tasks: - - name: Fetch topology {{ topo_file_loc }} - fetch: - src: "{{ topo_file_loc }}" - dest: "{{ topo_file_dest }}" - flat: yes - - - name: Read topology - set_fact: - topo_dict: "{{ lookup('file', topo_file_dest) | from_yaml }}" - - - include_tasks: "{{ set_ip_file }}" - loop: "{{ topo_dict.links }}" - - - name: Set /etc/hosts - become: yes - lineinfile: - path: /etc/hosts - line: "{{ item.ip }} {{ item.name }}" - state: present - loop: "{{ topo_dict.hosts.values() | list }}" - - - name: Add entries to ansible inventory at {{ remote_ansible_inventory }} - lineinfile: - path: "{{ remote_ansible_inventory }}" - insertbefore: BOF - line: "{{ item }}" - state: present - loop: "{{ topo_dict.hosts.keys() | list }}" - - - name: Add localhost to {{ remote_ansible_inventory }} - lineinfile: - path: "{{ remote_ansible_inventory }}" - insertbefore: BOF - line: "localhost" - state: present diff --git a/playbooks/mininet/templates/local_inventory.ini.j2 b/playbooks/mininet/templates/local_inventory.ini.j2 deleted file mode 100644 index 74ee52d5..00000000 --- a/playbooks/mininet/templates/local_inventory.ini.j2 +++ /dev/null @@ -1,20 +0,0 @@ -{{ public_ip }} -[all:vars] -remote_inventory_file={{ remote_inventory_file | default('/home/ubuntu/transparent-security.ini')}} -src_dir={{ src_dir | default('/home/ubuntu/transparent-security') }} -remote_srvc_log_dir={{ remote_srvc_log_dir | default('/var/log/transparent-security') }} -service_log_level={{ service_log_level | default('INFO') }} -log_dir={{ remote_srvc_log_dir | default('/var/log/transparent-security') }} -trans_sec_dir={{ trans_sec_dir | default('/home/ubuntu/transparent-security') }} -remote_scripts_dir={{ remote_scripts_dir | default('/etc/transparent-security') }} -topo_file={{ topo_file_loc | default('/etc/transparent-security/mininet-sim-topology.yaml') }} -forwarding_daemon_file={{ forwarding_daemon_file | default('/etc/transparent-security/forwarding-daemons.yml') }} -mn_host={{ mn_host | default('localhost') }} -sdn_host={{ sdn_host | default('localhost') }} -sdn_port={{ sdn_port | default('9998') }} -ae_host={{ ae_host | default('localhost') }} -sdn_dev_intf={{ sdn_dev_intf | default('lo') }} -ae_dev_intf={{ ae_dev_intf | default('lo') }} -ae_monitor_intf={{ ae_monitor_intf | default('core-eth3') }} -clone_egress_port={{ clone_egress_port | default('3') }} -scenario_name={{ scenario_name | default('full') }} diff --git a/playbooks/mininet/templates/topology_template-full.yml.j2 b/playbooks/mininet/templates/topology_template-full.yml.j2 deleted file mode 100755 index 7643cc0f..00000000 --- a/playbooks/mininet/templates/topology_template-full.yml.j2 +++ /dev/null @@ -1,284 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -# IP values must be the third address in an IPv4 /30 CIDR block for running CI -hosts: - Camera1: - name: Camera1 - id: 5 - switch_ip: 192.168.1.1 - switch_mac: 00:00:01:01:01:00 - ip: 192.168.1.2 - ipv6: 0000:0000:0000:0000:0000:0001:0001:0002 - ip_port: any - mac: 00:00:00:00:01:01 - type: webcam - NAS1: - name: NAS1 - id: 6 - switch_ip: 192.168.1.5 - switch_mac: 00:00:01:01:02:00 - ip: 192.168.1.6 - ipv6: 0000:0000:0000:0000:0000:0001:0001:0003 - ip_port: any - mac: 00:00:00:00:01:02 - type: nas - Game1: - name: Game1 - id: 7 - switch_ip: 192.168.1.9 - switch_mac: 00:00:01:01:03:00 - ip: 192.168.1.10 - ipv6: 0000:0000:0000:0000:0000:0001:0001:0004 - ip_port: any - mac: 00:00:00:00:01:03 - type: console - Camera2: - name: Camera2 - id: 8 - switch_ip: 192.168.2.1 - switch_mac: 00:00:02:02:01:00 - ipv6: 0000:0000:0000:0000:0000:0001:0002:0002 - ip: 192.168.2.2 - ip_port: any - mac: 00:00:00:00:02:01 - type: webcam - Game2: - name: Game2 - id: 9 - switch_ip: 192.168.2.5 - switch_mac: 00:00:02:02:02:00 - ip: 192.168.2.6 - ipv6: 0000:0000:0000:0000:0000:0001:0002:0003 - ip_port: any - mac: 00:00:00:00:02:02 - type: console - Camera3: - name: Camera3 - id: 10 - switch_ip: 192.168.3.1 - switch_mac: 00:00:03:03:01:00 - ip: 192.168.3.2 - ipv6: 0000:0000:0000:0000:0000:0001:0003:0002 - ip_port: any - mac: 00:00:00:00:03:01 - type: webcam - Game3: - name: Game3 - id: 11 - switch_ip: 192.168.3.5 - switch_mac: 00:00:03:03:02:00 - ip: 192.168.3.6 - ipv6: 0000:0000:0000:0000:0000:0001:0003:0003 - ip_port: any - mac: 00:00:00:00:03:02 - type: console - inet: - name: inet - id: 12 - switch_ip: 172.32.1.4 - ip: 172.32.1.5 - ipv6: 0000:0000:0000:0000:0000:0002:0001:0002 - ip_port: any - mac: 00:00:00:00:05:07 - type: target-server -switches: -# TODO - Determine why changes to the ID in any of the switch id values causes a GRPC exception in the SDN controller upon startup - gateway1: - name: gateway1 - id: 2 - runtime_p4info: {{ remote_scripts_dir }}/p4/gateway.p4info - runtime_json: {{ remote_scripts_dir }}/p4/gateway.json - public_ip: 172.32.1.1 - type: gateway - mac: 00:00:00:02:01:00 - grpc: {{ mn_host_ip }}:50053 - ipv6_term_host: inet - multicast_entries: - - egress_port: 1 - instance: 1 - - egress_port: 2 - instance: 1 - - egress_port: 3 - instance: 1 - - egress_port: 4 - instance: 1 - gateway2: - name: gateway2 - id: 3 - runtime_p4info: {{ remote_scripts_dir }}/p4/gateway.p4info - runtime_json: {{ remote_scripts_dir }}/p4/gateway.json - public_ip: 172.32.1.2 - type: gateway - mac: 00:00:00:02:02:00 - grpc: {{ mn_host_ip }}:50054 - ipv6_term_host: inet - multicast_entries: - - egress_port: 1 - instance: 1 - - egress_port: 2 - instance: 1 - - egress_port: 3 - instance: 1 - gateway3: - name: gateway3 - id: 4 - runtime_p4info: {{ remote_scripts_dir }}/p4/gateway.p4info - runtime_json: {{ remote_scripts_dir }}/p4/gateway.json - public_ip: 172.32.1.3 - type: gateway - mac: 00:00:00:02:03:00 - grpc: {{ mn_host_ip }}:50055 - ipv6_term_host: inet - multicast_entries: - - egress_port: 1 - instance: 1 - - egress_port: 2 - instance: 1 - - egress_port: 3 - instance: 1 - aggregate: - name: aggregate - id: 0 - runtime_p4info: {{ remote_scripts_dir }}/p4/aggregate.p4info - runtime_json: {{ remote_scripts_dir }}/p4/aggregate.json - type: aggregate - mac: 00:00:00:02:04:00 - grpc: {{ mn_host_ip }}:50051 - ipv6_term_host: inet - multicast_entries: - - egress_port: 1 - instance: 1 - - egress_port: 2 - instance: 1 - - egress_port: 3 - instance: 1 - - egress_port: 4 - instance: 1 - core: - name: core - id: 1 - runtime_p4info: {{ remote_scripts_dir }}/p4/core.p4info - runtime_json: {{ remote_scripts_dir }}/p4/core.json - clone_egress: {{ clone_egress_port }} - public_ip: 172.32.1.4 - telemetry_rpt: - type: external - name: ae - type: core - mac: 00:00:00:03:05:00 - grpc: {{ mn_host_ip }}:50052 - ipv6_term_host: inet - multicast_entries: - - egress_port: 1 - instance: 1 - - egress_port: 2 - instance: 1 -external: - ae: - name: ae - id: {{ ae_dev_intf }} - ip: {{ ae_ip }} - ip_port: 0 - mac: {{ switch_mac }} - sdn: - name: sdn - id: {{ sdn_dev_intf }} - ip: {{ sdn_ip }} - mac: {{ switch_mac }} -links: - - north_node: gateway1 - south_node: Camera1 - south_facing_port: 1 - south_facing_mac: 00:00:01:01:01:00 - south_facing_ip: 192.168.1.1 - latency: 0ms - bandwidth: null - - north_node: gateway1 - south_node: NAS1 - south_facing_port: 2 - south_facing_mac: 00:00:01:01:02:00 - south_facing_ip: 192.168.1.5 - latency: 0ms - bandwidth: null - - north_node: gateway1 - south_node: Game1 - south_facing_port: 3 - south_facing_mac: 00:00:01:01:03:00 - south_facing_ip: 192.168.1.9 - latency: 0ms - bandwidth: null - - north_node: gateway2 - south_node: Camera2 - south_facing_port: 1 - south_facing_mac: 00:00:02:02:01:00 - south_facing_ip: 192.168.2.1 - latency: 0ms - bandwidth: null - - north_node: gateway2 - south_node: Game2 - south_facing_port: 2 - south_facing_mac: 00:00:02:02:02:00 - south_facing_ip: 192.168.2.5 - latency: 0ms - bandwidth: null - - north_node: gateway3 - south_node: Camera3 - south_facing_port: 1 - south_facing_mac: 00:00:03:03:01:00 - south_facing_ip: 192.168.3.1 - latency: 0ms - bandwidth: null - - north_node: gateway3 - south_node: Game3 - south_facing_port: 2 - south_facing_mac: 00:00:03:03:02:00 - south_facing_ip: 192.168.3.5 - latency: 0ms - bandwidth: null - - north_node: aggregate - south_node: gateway1 - south_facing_port: 1 - north_facing_port: 4 - latency: 0ms - bandwidth: null - - north_node: aggregate - south_node: gateway2 - south_facing_port: 2 - north_facing_port: 3 - latency: 0ms - bandwidth: null - - north_node: aggregate - south_node: gateway3 - south_facing_port: 3 - north_facing_port: 3 - latency: 0ms - bandwidth: null - - north_node: core - south_node: aggregate - south_facing_port: 1 - north_facing_port: 4 - latency: 0ms - bandwidth: null - - north_node: inet - south_node: core - north_facing_port: 2 - latency: 0ms - bandwidth: null - - north_node: ae - south_node: core - north_facing_port: 3 - latency: 0ms - bandwidth: null - l2ptr: 53261 diff --git a/playbooks/mininet/templates/topology_template-lab_trial.yml.j2 b/playbooks/mininet/templates/topology_template-lab_trial.yml.j2 deleted file mode 100644 index 9b8837b8..00000000 --- a/playbooks/mininet/templates/topology_template-lab_trial.yml.j2 +++ /dev/null @@ -1,121 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -# IP values must be the third address in an IPv4 /30 CIDR block for running CI -hosts: - host1: - name: host1 - id: 5 - switch_ip: 192.168.1.1 - ip: 192.168.1.2 - ipv6: 0000:0000:0000:0000:0000:0001:0001:0002 - ip_port: any - mac: 00:00:00:00:01:01 - type: webcam - host2: - name: host2 - id: 6 - switch_ip: 192.168.1.5 - ip: 192.168.1.6 - ipv6: 0000:0000:0000:0000:0000:0001:0001:0003 - ip_port: any - mac: 00:00:00:00:01:02 - type: nas - inet: - name: inet - id: 12 - switch_ip: 192.168.1.9 - ip: 192.168.1.10 - ipv6: 0000:0000:0000:0000:0000:0002:0001:0002 - ip_port: any - mac: 00:00:00:00:05:07 - type: target-server - ae: - name: ae - id: 13 - switch_ip: 10.0.1.8 - ip: 10.0.1.9 - ipv6: 0000:0000:0000:0000:0000:0002:0001:0003 - ip_port: any - mac: 00:00:00:00:05:08 - type: server -switches: - aggregate: - name: aggregate - id: 0 - runtime_p4info: {{ remote_scripts_dir }}/p4/aggregate.p4info - runtime_json: {{ remote_scripts_dir }}/p4/aggregate.json - type: aggregate - mac: 00:00:00:02:04:00 - grpc: 127.0.0.1:50051 - ipv6_term_host: inet - multicast_entries: - - egress_port: 1 - instance: 1 - - egress_port: 2 - instance: 1 - - egress_port: 3 - instance: 1 - core: - name: core - id: 1 - runtime_p4info: {{ remote_scripts_dir }}/p4/core.p4info - runtime_json: {{ remote_scripts_dir }}/p4/core.json - clone_egress: 3 - telemetry_rpt: - type: host - name: ae - type: core - mac: 00:00:00:03:05:00 - grpc: 127.0.0.1:50052 - ipv6_term_host: inet - multicast_entries: - - egress_port: 1 - instance: 1 - - egress_port: 2 - instance: 1 - - egress_port: 3 - instance: 1 -links: - - north_node: aggregate - south_node: host1 - south_facing_port: 1 - south_facing_ip: 192.168.1.1 - latency: 0ms - bandwidth: null - - north_node: aggregate - south_node: host2 - south_facing_port: 2 - south_facing_ip: 192.168.1.5 - latency: 0ms - bandwidth: null - - north_node: core - south_node: aggregate - south_facing_port: 1 - north_facing_port: 3 - latency: 0ms - bandwidth: null - - north_node: inet - south_node: core - north_facing_port: 2 - north_facing_ip: 192.168.1.9 - latency: 0ms - bandwidth: null - - north_node: ae - south_node: core - north_facing_port: 3 - north_facing_ip: 10.0.1.8 - latency: 0ms - bandwidth: null - l2ptr: 53261 diff --git a/playbooks/mininet/templates/topology_template-single_switch.yml.j2 b/playbooks/mininet/templates/topology_template-single_switch.yml.j2 deleted file mode 100644 index 556ddc95..00000000 --- a/playbooks/mininet/templates/topology_template-single_switch.yml.j2 +++ /dev/null @@ -1,88 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- - -# IP values must be the third address in an IPv4 /30 CIDR block for running CI -hosts: - host1: - name: host1 - id: 1 - switch_ip: 192.168.1.1 - ip: 192.168.1.2 - ipv6: 0000:0000:0000:0000:0000:0001:0001:0002 - ip_port: any - mac: 00:00:00:00:01:01 - type: webcam - host2: - name: host2 - id: 2 - switch_ip: 172.32.1.4 - ip: 172.32.1.5 - ipv6: 0000:0000:0000:0000:0000:0002:0001:0002 - ip_port: any - mac: 00:00:00:00:05:07 - type: target-server -{% if p4_progs[0] != 'aggregate' %} - clone: - name: clone - id: 3 - switch_ip: 10.0.1.8 - ip: 10.0.1.9 - ipv6: 0000:0000:0000:0000:0000:0002:0001:0003 - ip_port: any - mac: 00:00:00:00:05:08 - type: server -{% endif %} -switches: - {{ p4_progs[0] }}: - name: {{ p4_progs[0] }} - id: 0 - runtime_p4info: {{ remote_scripts_dir }}/p4/{{ p4_progs[0] }}.p4info - runtime_json: {{ remote_scripts_dir }}/p4/{{ p4_progs[0] }}.json - clone_egress: {{ clone_egress_port }} - public_ip: 172.32.1.4 - subnet: 10.0.1.0 - type: {{ p4_progs[0] }} - mac: 00:00:00:01:01:00 - grpc: {{ mn_host_ip }}:50051 - telemetry_rpt: - type: host - name: clone - ipv6_term_host: host2 -{% if p4_progs[0] != 'gateway' %} - multicast_entries: - - egress_port: 1 - instance: 1 - - egress_port: 2 - instance: 1 - - egress_port: 3 - instance: 1 -{% endif %} -links: - - north_node: {{ p4_progs[0] }} - south_node: host1 - south_facing_port: 1 - latency: 0ms - bandwidth: null - - north_node: host2 - south_node: {{ p4_progs[0] }} - north_facing_port: 2 - latency: 0ms - bandwidth: null -{% if p4_progs[0] != 'aggregate' %} - - north_node: clone - south_node: {{ p4_progs[0] }} - north_facing_port: 3 - latency: 0ms - bandwidth: null -{% endif %} diff --git a/playbooks/scenarios/full/all-tofino.yml b/playbooks/scenarios/full/all-tofino.yml deleted file mode 100644 index 24e5613b..00000000 --- a/playbooks/scenarios/full/all-tofino.yml +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Simple scenario where packets are sent through 3 devices and only the last -# one will be demonstrating dropped packets ---- -# Discover routes for full scenario -- import_playbook: discover_full.yml - vars: - host: gateway1 - interfaces: - - veth3 - -- import_playbook: discover_full.yml - vars: - host: gateway2 - interfaces: - - veth2 - -- import_playbook: discover_full.yml - vars: - host: gateway3 - interfaces: - - veth2 - -# UDP Flood Scenario IPv4 -- import_playbook: packet-flood.yml - vars: - scenario_send_protocol: UDP - scenario_send_ip_version: 4 - - -# UDP Flood Scenario IPv6 -- import_playbook: packet-flood.yml - vars: - scenario_send_protocol: UDP - scenario_send_ip_version: 6 - -# SYN Flood Scenario IPv4 -- import_playbook: packet-flood.yml - vars: - scenario_send_protocol: TCP - scenario_send_ip_version: 4 - -# SYN Flood Scenario IPv6 -- import_playbook: packet-flood.yml - vars: - scenario_send_protocol: TCP - scenario_send_ip_version: 6 diff --git a/playbooks/scenarios/full/all.yml b/playbooks/scenarios/full/all.yml deleted file mode 100644 index 4ea602f1..00000000 --- a/playbooks/scenarios/full/all.yml +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Simple scenario where packets are sent through 3 devices and only the last -# one will be demonstrating dropped packets ---- -# Discover routes for full scenario -- import_playbook: discover_full.yml - vars: - host: localhost - interfaces: - - gateway1-eth4 - - gateway2-eth3 - - gateway3-eth3 - -# UDP Flood Scenario IPv4 -- import_playbook: packet-flood.yml - vars: - scenario_send_protocol: UDP - scenario_send_ip_version: 4 - -# UDP Flood Scenario IPv6 -- import_playbook: packet-flood.yml - vars: - scenario_send_protocol: UDP - scenario_send_ip_version: 6 - -# SYN Flood Scenario IPv4 -- import_playbook: packet-flood.yml - vars: - scenario_send_protocol: TCP - scenario_send_ip_version: 4 - -# SYN Flood Scenario IPv6 -- import_playbook: packet-flood.yml - vars: - scenario_send_protocol: TCP - scenario_send_ip_version: 6 diff --git a/playbooks/scenarios/full/discover_full.yml b/playbooks/scenarios/full/discover_full.yml deleted file mode 100644 index 81da20a5..00000000 --- a/playbooks/scenarios/full/discover_full.yml +++ /dev/null @@ -1,17 +0,0 @@ ---- -- hosts: "{{ host }}" - gather_facts: no - tasks: - - name: Read in topology - set_fact: - topo_dict: "{{ lookup('file','{{ topo_file_loc }}') | from_yaml }}" - - - name: Ping inet host - include_tasks: ../test_cases/ping_from.yml - vars: - ip: "{{ topo_dict.hosts['inet'].ip }}" - intf: "{{ item }}" - count: 3 - interval: .5 - pause_time: 0 - loop: "{{ interfaces }}" diff --git a/playbooks/scenarios/full/packet-flood.yml b/playbooks/scenarios/full/packet-flood.yml deleted file mode 100644 index ebb188d5..00000000 --- a/playbooks/scenarios/full/packet-flood.yml +++ /dev/null @@ -1,128 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Simple scenario where packets are sent through 3 devices and only the last -# one will be demonstrating dropped packets ---- -# Sending 150 packets within a second 2x through gateway 1. -# The second set should be dropped else this will fail -- import_playbook: ../test_cases/send_receive.yml - vars: - send_protocol: "{{ scenario_send_protocol | default('UDP') }}" - ip_version: "{{ scenario_send_ip_version | default(4) }}" - topo_dict: "{{ lookup('file','{{ topo_file_loc }}') | from_yaml }}" - switch_name: gateway1 - core_switch_name: core - remote_send_host: Camera1 - remote_rec_host: inet - switch: "{{ topo_dict.switches[switch_name] }}" - send_host: "{{ remote_send_host }}" - rec_host: "{{ remote_rec_host }}" - sender: "{{ topo_dict.hosts[remote_send_host] }}" - receiver: "{{ topo_dict.hosts[remote_rec_host] }}" - switch_egress_intf: eth4 - dst_mac: "{{ sender['switch_mac'] }}" - receiver_log_filename: "{{ send_protocol }}-flood-receiver-{{ rec_host }}-{{ send_protocol }}-1.out" - sender_log_filename: "{{ send_protocol }}-flood-sender-{{ send_host }}-{{ send_protocol }}-1.out" - send_port: "{{ '%s' % range(2000, 10000) | random(seed=now|int + 1) }}" - send_src_port: "{{ '%s' % range(2000, 10000) | random(seed=now|int + 2) }}" - send_packet_count: 150 - min_received_packet_count: 99 - max_received_packet_count: "{{ send_packet_count }}" - send_loops: 2 - send_loop_delay: 5 - cleanup_rest_call: - url: "http://{{ sdn_ip }}:{{ sdn_port }}/gwAttack" - body: - src_ip: "{% if scenario_send_ip_version|int == 4 %}{{ sender.ip }}{% else %}{{ sender.ipv6 }}{% endif %}" - packet_size: 112 - attack_type: "{{ send_protocol }} Flood" - dst_port: "{{ send_port }}" - dst_ip: "{% if scenario_send_ip_version|int == 4 %}{{ receiver.ip }}{% else %}{{ receiver.ipv6 }}{% endif %}" - src_mac: "{{ sender.mac }}" - ok_status: 201 - -# Sending 150 packets within a second 2x through gateway 2. -# The second set should be dropped else this will fail -- import_playbook: ../test_cases/send_receive.yml - vars: - send_protocol: "{{ scenario_send_protocol | default('UDP') }}" - ip_version: "{{ scenario_send_ip_version | default(4) }}" - topo_dict: "{{ lookup('file','{{ topo_file_loc }}') | from_yaml }}" - switch_name: gateway2 - core_switch_name: core - remote_send_host: Camera2 - remote_rec_host: inet - switch: "{{ topo_dict.switches[switch_name] }}" - send_host: "{{ remote_send_host }}" - rec_host: "{{ remote_rec_host }}" - sender: "{{ topo_dict.hosts[remote_send_host] }}" - receiver: "{{ topo_dict.hosts[remote_rec_host] }}" - switch_egress_intf: eth3 - dst_mac: "{{ sender['switch_mac'] }}" - receiver_log_filename: "{{ send_protocol }}-flood-receiver-{{ rec_host }}-{{ send_protocol }}-2.out" - sender_log_filename: "{{ send_protocol }}-flood-sender-{{ send_host }}-{{ send_protocol }}-2.out" - send_port: "{{ '%s' % range(2000, 10000) | random(seed=now|int + 3) }}" - send_src_port: "{{ '%s' % range(2000, 10000) | random(seed=now|int + 4) }}" - send_packet_count: 150 - min_received_packet_count: 99 - max_received_packet_count: "{{ send_packet_count }}" - send_loops: 2 - send_loop_delay: 5 - cleanup_rest_call: - url: "http://{{ sdn_ip }}:{{ sdn_port }}/gwAttack" - body: - src_ip: "{% if scenario_send_ip_version|int == 4 %}{{ sender.ip }}{% else %}{{ sender.ipv6 }}{% endif %}" - packet_size: 112 - attack_type: "{{ send_protocol }} Flood" - dst_port: "{{ send_port }}" - dst_ip: "{% if scenario_send_ip_version|int == 4 %}{{ receiver.ip }}{% else %}{{ receiver.ipv6 }}{% endif %}" - src_mac: "{{ sender.mac }}" - ok_status: 201 - -# Sending 150 packets within a second 2x through gateway 3. -# The second set should be dropped else this will fail -- import_playbook: ../test_cases/send_receive.yml - vars: - send_protocol: "{{ scenario_send_protocol | default('UDP') }}" - ip_version: "{{ scenario_send_ip_version | default(4) }}" - topo_dict: "{{ lookup('file','{{ topo_file_loc }}') | from_yaml }}" - switch_name: gateway3 - core_switch_name: core - remote_send_host: Camera3 - remote_rec_host: inet - switch: "{{ topo_dict.switches[switch_name] }}" - send_host: "{{ remote_send_host }}" - rec_host: "{{ remote_rec_host }}" - sender: "{{ topo_dict.hosts[remote_send_host] }}" - receiver: "{{ topo_dict.hosts[remote_rec_host] }}" - switch_egress_intf: eth3 - dst_mac: "{{ sender['switch_mac'] }}" - receiver_log_filename: "{{ send_protocol }}-flood-receiver-{{ rec_host }}-{{ send_protocol }}-3.out" - sender_log_filename: "{{ send_protocol }}-flood-sender-{{ send_host }}-{{ send_protocol }}-3.out" - send_port: "{{ '%s' % range(2000, 10000) | random(seed=now|int + 5) }}" - send_src_port: "{{ '%s' % range(2000, 10000) | random(seed=now|int + 6) }}" - send_packet_count: 150 - min_received_packet_count: 99 - max_received_packet_count: "{{ send_packet_count }}" - send_loops: 2 - send_loop_delay: 5 - cleanup_rest_call: - url: "http://{{ sdn_ip }}:{{ sdn_port }}/gwAttack" - body: - src_ip: "{% if scenario_send_ip_version|int == 4 %}{{ sender.ip }}{% else %}{{ sender.ipv6 }}{% endif %}" - packet_size: 112 - attack_type: "{{ send_protocol }} Flood" - dst_port: "{{ send_port }}" - dst_ip: "{% if scenario_send_ip_version|int == 4 %}{{ receiver.ip }}{% else %}{{ receiver.ipv6 }}{% endif %}" - src_mac: "{{ sender.mac }}" - ok_status: 201 diff --git a/playbooks/scenarios/gateway/all.yml b/playbooks/scenarios/gateway/all.yml deleted file mode 100644 index dff8d096..00000000 --- a/playbooks/scenarios/gateway/all.yml +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Simple scenario where packets are sent through 3 devices and only the last -# one will be demonstrating dropped packets ---- -# Data Forward scenarios for UDP and IPv4 -- import_playbook: data-forward.yml - vars: - scenario_send_protocol: UDP - scenario_send_ip_version: 4 - -# Data Forward scenarios for UDP and IPv6 -- import_playbook: data-forward.yml - vars: - scenario_send_protocol: UDP - scenario_send_ip_version: 6 - -# Data Forward scenarios for TCP and IPv4 -- import_playbook: data-forward.yml - vars: - scenario_send_protocol: TCP - scenario_send_ip_version: 4 - -# Data Forward scenarios for TCP and IPv6 -- import_playbook: data-forward.yml - vars: - scenario_send_protocol: TCP - scenario_send_ip_version: 6 - -# Data Drop scenarios for UDP and IPv4 -- import_playbook: data-drop.yml - vars: - scenario_send_protocol: UDP - scenario_send_ip_version: 4 - -# Data Drop scenarios for UDP and IPv6 -- import_playbook: data-drop.yml - vars: - scenario_send_protocol: UDP - scenario_send_ip_version: 6 - -# Data Drop scenarios for TCP and IPv4 -- import_playbook: data-drop.yml - vars: - scenario_send_protocol: TCP - scenario_send_ip_version: 4 - -# Data Drop scenarios for TCP and IPv6 -- import_playbook: data-drop.yml - vars: - scenario_send_protocol: TCP - scenario_send_ip_version: 6 - -# Data Inspection scenarios for UDP and IPv4 -- import_playbook: data-inspection.yml - vars: - scenario_send_protocol: UDP - scenario_send_ip_version: 4 - -# Data Inspection scenarios for UDP and IPv6 -- import_playbook: data-inspection.yml - vars: - scenario_send_protocol: UDP - scenario_send_ip_version: 6 - -# Data Inspection scenarios for TCP and IPv4 -- import_playbook: data-inspection.yml - vars: - scenario_send_protocol: TCP - scenario_send_ip_version: 4 - -# Data Inspection scenarios for TCP and IPv6 -- import_playbook: data-inspection.yml - vars: - scenario_send_protocol: TCP - scenario_send_ip_version: 6 diff --git a/playbooks/scenarios/gateway/data-drop.yml b/playbooks/scenarios/gateway/data-drop.yml deleted file mode 100644 index 6278807e..00000000 --- a/playbooks/scenarios/gateway/data-drop.yml +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Simple scenario where packets are sent through 3 devices and only the last -# one will be demonstrating dropped packets ---- - -# Basic Data Drop scenario -- import_playbook: ../general/data_drop_basic.yml - vars: - send_protocol: "{{ scenario_send_protocol | default('UDP') }}" - ip_version: "{{ scenario_send_ip_version | default(4) }}" - topo_dict: "{{ lookup('file','{{ topo_file_loc }}') | from_yaml }}" - switch: "{{ topo_dict.switches.gateway }}" - remote_send_host: host1 - remote_rec_host: host2 - send_host: "{{ remote_send_host }}" - rec_host: "{{ remote_rec_host }}" - send_port: 4321 - sender: "{{ topo_dict.hosts[remote_send_host] }}" - receiver: "{{ topo_dict.hosts[remote_rec_host] }}" - data_forwards: - - url: "http://{{ sdn_ip }}:{{ sdn_port }}/dataForward" - body: - device_id: "{{ switch.id }}" - switch_mac: "{{ switch.mac }}" - dst_mac: "{{ receiver.mac }}" - output_port: 2 - ok_status: 201 - attack: - url: "http://{{ sdn_ip }}:{{ sdn_port }}/gwAttack" - body: - src_ip: "{% if ip_version|int == 4 %}{{ sender.ip }}{% else %}{{ sender.ipv6 }}{% endif %}" - packet_size: 112 - attack_type: "UDP Flood" - dst_port: "{{ send_port }}" - dst_ip: "{% if ip_version|int == 4 %}{{ receiver.ip }}{% else %}{{ receiver.ipv6 }}{% endif %}" - src_mac: "{{ sender.mac }}" - ok_status: 201 diff --git a/playbooks/scenarios/gateway/data-forward.yml b/playbooks/scenarios/gateway/data-forward.yml deleted file mode 100644 index 975eebdb..00000000 --- a/playbooks/scenarios/gateway/data-forward.yml +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Simple scenario where packets are sent through 3 devices and only the last -# one will be demonstrating dropped packets ---- - -# Basic Data Forward scenario for UDP -- import_playbook: ../general/data_forward_basic.yml - vars: - send_protocol: "{{ scenario_send_protocol | default('UDP') }}" - ip_version: "{{ scenario_send_ip_version | default(4) }}" - topo_dict: "{{ lookup('file','{{ topo_file_loc }}') | from_yaml }}" - switch: "{{ topo_dict.switches.gateway }}" - send_host: host1 - rec_host: host2 - sender: "{{ topo_dict.hosts[send_host] }}" - receiver: "{{ topo_dict.hosts[rec_host] }}" - data_forwards: - - url: "http://{{ sdn_ip }}:{{ sdn_port }}/dataForward" - body: - device_id: "{{ switch.id }}" - switch_mac: "{{ switch.mac }}" - dst_mac: "{{ receiver.mac }}" - output_port: 2 - ok_status: 201 diff --git a/playbooks/scenarios/gateway/data-inspection.yml b/playbooks/scenarios/gateway/data-inspection.yml deleted file mode 100644 index be8f1e5f..00000000 --- a/playbooks/scenarios/gateway/data-inspection.yml +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Simple scenario where packets are sent through 3 devices and only the last -# one will be demonstrating dropped packets ---- - -# Basic Data Inspection scenario for UDP -- import_playbook: ../single_switch/data_inspection_basic.yml - vars: - send_protocol: "{{ scenario_send_protocol | default('UDP') }}" - ip_version: "{{ scenario_send_ip_version | default(4) }}" - topo_dict: "{{ lookup('file','{{ topo_file_loc }}') | from_yaml }}" - switch: "{{ topo_dict.switches.gateway }}" - remote_send_host: host1 - remote_rec_host: host2 - send_host: "{{ remote_send_host }}" - rec_host: "{{ remote_rec_host }}" - sender: "{{ topo_dict.hosts[remote_send_host] }}" - receiver: "{{ topo_dict.hosts[remote_rec_host] }}" - ws_calls: - - url: "http://{{ sdn_ip }}:{{ sdn_port }}/dataForward" - body: - device_id: "{{ switch.id }}" - switch_mac: "{{ switch.mac }}" - dst_mac: "{{ receiver.mac }}" - output_port: 2 - ok_status: 201 - data_inspections: - - url: "http://{{ sdn_ip }}:{{ sdn_port }}/dataInspection" - body: - device_id: "{{ switch.id }}" - switch_mac: "{{ switch.mac }}" - device_mac: "{{ sender.mac }}" - ok_status: 201 - sr1_rec_int_hops: 1 diff --git a/playbooks/scenarios/general/drop_report_simple.yml b/playbooks/scenarios/general/drop_report_simple.yml index f6c32efa..dd3d4015 100644 --- a/playbooks/scenarios/general/drop_report_simple.yml +++ b/playbooks/scenarios/general/drop_report_simple.yml @@ -85,7 +85,7 @@ command: "{{ receive_cmd }}" register: receive_out changed_when: receive_out is not failed - retries: 3 + retries: 6 delay: 10 until: receive_out.stdout | length > 5 and receive_out.stdout | from_json | length == send_ports | length diff --git a/playbooks/scenarios/lab_trial/all-pkt-flood.yml b/playbooks/scenarios/lab_trial/all-pkt-flood.yml index 4a852131..90c873cb 100644 --- a/playbooks/scenarios/lab_trial/all-pkt-flood.yml +++ b/playbooks/scenarios/lab_trial/all-pkt-flood.yml @@ -16,16 +16,33 @@ # Start AE service - hosts: ae gather_facts: no - become: yes tasks: - name: Start tps-tofino-ae service + become: yes systemd: name: tps-tofino-ae - state: restarted + state: started - - name: Wait 10 seconds for tps-tofino-ae to fully start - pause: - seconds: 10 + - name: Wait for K8s deployment/trpt-to-kafka-0 to become available + command: kubectl wait --for=condition=available --timeout=600s deployment/trpt-to-kafka-0 -n default + register: trpt_to_kafka_check + retries: 10 + delay: 20 + until: trpt_to_kafka_check is not failed + + - name: Wait for K8s deployment/kafka-trpt-ddos-detect-0 to become available + command: kubectl wait --for=condition=available --timeout=600s deployment/kafka-trpt-ddos-detect-0 -n default + register: ddos_check + retries: 5 + delay: 10 + until: ddos_check is not failed + + - name: Wait for K8s deployment/kafka-trpt-drop-clear-0 to become available + command: kubectl wait --for=condition=available --timeout=600s deployment/kafka-trpt-drop-clear-0 -n default + register: drop_check + retries: 5 + delay: 10 + until: drop_check is not failed # Packet Flood scenarios for UDP and IPv4 - import_playbook: pkt-flood.yml diff --git a/playbooks/scenarios/lab_trial/all.yml b/playbooks/scenarios/lab_trial/all.yml index c9ec0871..677630e4 100644 --- a/playbooks/scenarios/lab_trial/all.yml +++ b/playbooks/scenarios/lab_trial/all.yml @@ -27,11 +27,11 @@ # Data Drop scenarios - import_playbook: all-data-drop.yml +# Switches now operational - Run Packet Flood scenarios to test the AE +- import_playbook: all-pkt-flood.yml + # Drop Reporting scenarios - import_playbook: all-drop-rpt.yml # Packet Performance tests - import_playbook: iperf.yml - -# Switches now operational - Run Packet Flood scenarios to test the AE -- import_playbook: all-pkt-flood.yml diff --git a/playbooks/scenarios/test_cases/write_clone_entry.yml b/playbooks/scenarios/test_cases/write_clone_entry.yml deleted file mode 100644 index 7df48b88..00000000 --- a/playbooks/scenarios/test_cases/write_clone_entry.yml +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Simple scenario where packets are sent through 3 devices and only the last -# one will be demonstrating dropped packets ---- -- hosts: localhost - gather_facts: no - vars: - topo_dict: "{{ lookup('file','{{ topo_file_loc }}') | from_yaml }}" - switch: "{{ topo_dict.switches.core }}" - log_dir: "{{ log_dir }}" - log_file: write_to_clone.log - clone: "{{ clone_entry | default('True') }}" - tasks: - - name: Create clone session entry command - set_fact: - add_clone_entry_cmd: > - {{ trans_sec_dir }}/bin/write_to_clone.py - -a {{ switch.grpc }} -d {{ switch.id }} -p {{ switch.runtime_p4info }} -l {{ log_dir }} - -lf {{ log_file }} -c {{ clone }} -ce {{ switch.clone_egress }} - - - name: Create log directory {{ log_dir }} - file: - path: "{{ log_dir }}" - state: directory - - - name: Build clone entry with command [{{ add_clone_entry_cmd }}] - command: "{{ add_clone_entry_cmd }}" - register: cmd_out - changed_when: cmd_out is not failed diff --git a/playbooks/scenarios/test_cases/write_to_p4_table.yml b/playbooks/scenarios/test_cases/write_to_p4_table.yml deleted file mode 100644 index 00154d76..00000000 --- a/playbooks/scenarios/test_cases/write_to_p4_table.yml +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# integration test case where one host creates a packet sniffer and another -# sends packets. The validation routine counts the number received via the -# receiver's log file ---- - -# Adds a table entry to a P4 switch -- hosts: localhost - gather_facts: no - vars: - table_entry_config: "{{ te_conf_file }}" - log_dir: "{{ p4_te_log_dir }}" - log_file: "{{ p4_te_log_file }}" - insert_flag: "{{ p4_te_insert | default('True') }}" - template_file: ../templates/table_entry.yml.j2 - table_action: "{{ p4_table_action }}" - ingress_table: "{{ p4_ingress_table_flag | default('True') }}" - execute: "{{ execute_insert | default(True) }}" - tasks: - - name: Execute when - block: - - name: Create config {{ table_entry_config }} - template: - src: "{{ template_file }}" - dest: "{{ table_entry_config }}" - backup: yes - - - name: Creating add table command - set_fact: - add_table_entry_cmd: > - {{ trans_sec_dir }}/bin/write_to_p4_table.py - -s {{ switch.name }} - -t {{ topo_file_loc }} - -tc {{ table_entry_config }} - -l {{ log_dir }} - -lf {{ log_file }} - -n {{ ingress_table }} - - - name: Create log directory {{ log_dir }} - file: - path: "{{ log_dir }}" - state: directory - - - name: Applying requested table entry with command [{{ add_table_entry_cmd }}] with insert_flag - [{{ insert_flag }}] - command: "{{ add_table_entry_cmd }} -i {{ insert_flag }}" - register: cmd_out - changed_when: cmd_out is not failed - ignore_errors: yes - - - name: Deleting requested table entry with command [{{ add_table_entry_cmd }}] and election-id 2 - command: "{{ add_table_entry_cmd }} -i False" - when: cmd_out is failed and insert_flag == "True" - - - name: Try table entry with command [{{ add_table_entry_cmd }}] with election-id 2 - command: "{{ add_table_entry_cmd }} -e 2 -i {{ insert_flag }}" - when: cmd_out is failed - when: execute|bool diff --git a/tests/trans_sec/mininet/exercise_tests.py b/playbooks/siddhi/docker/Dockerfile similarity index 56% rename from tests/trans_sec/mininet/exercise_tests.py rename to playbooks/siddhi/docker/Dockerfile index 8cc2d8dd..57fb1c98 100644 --- a/tests/trans_sec/mininet/exercise_tests.py +++ b/playbooks/siddhi/docker/Dockerfile @@ -1,4 +1,4 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. +# Copyright (c) 2021 Cable Television Laboratories, Inc. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -11,13 +11,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging -import unittest +FROM siddhiio/siddhi-runner-alpine:5.1.2 -logger = logging.getLogger('exercise_tests') +RUN apk --update add net-tools tcpdump sudo +RUN apk update && apk add ca-certificates && rm -rf /var/cache/apk/* +RUN echo 'siddhi_user:siddhi' | chpasswd +RUN echo '%wheel ALL=(ALL) ALL' > /etc/sudoers.d/wheel +RUN adduser siddhi_user wheel - -class ExerciseTopoTests(unittest.TestCase): - """ - Stub for testing the ExerciseTopo class - """ +ARG classpath_lib=. +COPY $classpath_lib/lib/* /home/siddhi_user/siddhi-runner/lib/ diff --git a/playbooks/siddhi/kubernetes/templates/kafka-trpt-ddos-detection.yaml.j2 b/playbooks/siddhi/kubernetes/templates/kafka-trpt-ddos-detection.yaml.j2 new file mode 100644 index 00000000..a8dcb0b1 --- /dev/null +++ b/playbooks/siddhi/kubernetes/templates/kafka-trpt-ddos-detection.yaml.j2 @@ -0,0 +1,87 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: siddhi.io/v1alpha2 +kind: SiddhiProcess +metadata: + name: kafka-trpt-ddos-detect +spec: + apps: + - script: | + @App:name("KafkaSourcePacketJSON") + @source( + type="kafka", + topic.list="trptPacket", + bootstrap.servers="{{ kafka_svc_ip }}:9092", + group.id="test", + threading.option="single.thread", + @map( + type="p4-trpt", + @attributes( + src_mac="intHdr.mdStackHdr.origMac", + ip_ver="ipHdr.version", + dst_ip="ipHdr.dstAddr", + dst_port="protoHdr.dstPort" + ) + ) + ) + define stream trptPktStream ( + src_mac string, + ip_ver int, + dst_ip string, + dst_port long); + + @sink( + type="http", + publisher.url="http://{{ sdn_ip }}:9998/aggAttack", + method="POST", + headers="trp:headers", + @map(type="json") + ) + define stream attackStream ( + src_mac string, + ip_ver int, + dst_ip string, + dst_port long, + count long + ); + + @info(name = "trptJsonQuery") + from trptPktStream#window.time(1 sec) + select src_mac, ip_ver, dst_ip, dst_port, count(ip_ver) as count + group by src_mac, dst_ip, dst_port + having count == 10 + insert into attackStream; + + container: + image: "spisarski/siddhi" + imagePullPolicy: Always + + persistentVolumeClaim: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: standard + volumeMode: Filesystem + + runner: | + state.persistence: + enabled: true + intervalInMin: 5 + revisionsToKeep: 2 + persistenceStore: io.siddhi.distribution.core.persistence.FileSystemPersistenceStore + config: + location: siddhi-app-persistence diff --git a/playbooks/siddhi/kubernetes/templates/kafka-trpt-drop-clear.yaml.j2 b/playbooks/siddhi/kubernetes/templates/kafka-trpt-drop-clear.yaml.j2 new file mode 100644 index 00000000..3f80d227 --- /dev/null +++ b/playbooks/siddhi/kubernetes/templates/kafka-trpt-drop-clear.yaml.j2 @@ -0,0 +1,84 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: siddhi.io/v1alpha2 +kind: SiddhiProcess +metadata: + name: kafka-trpt-drop-clear +spec: + apps: + - script: | + @App:name("KafkaSourceDropJSON") + @source( + type="kafka", + topic.list="trptDrop", + bootstrap.servers="{{ kafka_svc_ip }}:9092", + group.id="test", + threading.option="single.thread", + @map( + type="p4-trpt", + @attributes( + timestamp="dropHdr.timestamp", + dropKey="dropHdr.dropKey", + dropCount="dropHdr.dropCount" + ) + ) + ) + define stream trptDropStream ( + timestamp long, + dropKey string, + dropCount long + ); + + @sink( + type="http", + publisher.url="http://{{ sdn_ip }}:9998/aggAttack", + method="DELETE", + headers="trp:headers", + @map(type="json") + ) + define stream dropAttackStream ( + dropKey string, + dropCount long, + count long + ); + + @info(name = "trptJsonQuery") + from trptDropStream#window.time(35 sec) + select dropKey, dropCount, count(dropCount) as count + group by dropKey, dropCount + having count >= 3 + insert into dropAttackStream; + + container: + image: "spisarski/siddhi" + imagePullPolicy: Always + + persistentVolumeClaim: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: standard + volumeMode: Filesystem + + runner: | + state.persistence: + enabled: true + intervalInMin: 5 + revisionsToKeep: 2 + persistenceStore: io.siddhi.distribution.core.persistence.FileSystemPersistenceStore + config: + location: siddhi-app-persistence diff --git a/playbooks/siddhi/kubernetes/templates/trpt-pv.yaml.j2 b/playbooks/siddhi/kubernetes/templates/trpt-pv.yaml.j2 new file mode 100644 index 00000000..f21afb6f --- /dev/null +++ b/playbooks/siddhi/kubernetes/templates/trpt-pv.yaml.j2 @@ -0,0 +1,15 @@ +kind: PersistentVolume +apiVersion: v1 +metadata: + name: siddhi-pv + labels: + type: local +spec: + storageClassName: standard + persistentVolumeReclaimPolicy: Delete + capacity: + storage: 1Gi + accessModes: + - ReadWriteOnce + hostPath: + path: "/home/siddhi_user/" diff --git a/playbooks/siddhi/kubernetes/templates/trpt-to-kafka.yaml.j2 b/playbooks/siddhi/kubernetes/templates/trpt-to-kafka.yaml.j2 new file mode 100644 index 00000000..8bb5a190 --- /dev/null +++ b/playbooks/siddhi/kubernetes/templates/trpt-to-kafka.yaml.j2 @@ -0,0 +1,84 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: siddhi.io/v1alpha2 +kind: SiddhiProcess +metadata: + name: trpt-to-kafka +spec: + apps: + - script: | + @App:name("UDPSourceTRPT") + @source( + type="udp", + listen.port="556", + @map( + type="p4-trpt", + @attributes( + in_type="telemRptHdr.inType", + full_json="jsonString" + ) + ) + ) + define stream trptUdpStream (in_type int, full_json object); + + @sink( + type="kafka", + topic="trptPacket", + bootstrap.servers="{{ kafka_svc_ip }}:9092", + is.binary.message = "false", + @map(type="text") + ) + define stream trptPacket (full_json object); + + @sink( + type="kafka", + topic="trptDrop", + bootstrap.servers="{{ kafka_svc_ip }}:9092", + is.binary.message = "false", + @map(type="text") + ) + define stream trptDrop (full_json object); + + @info(name = "TrptPacket") + from trptUdpStream[in_type != 2] + select full_json + insert into trptPacket; + + @info(name = "TrptDrop") + from trptUdpStream[in_type == 2] + select full_json + insert into trptDrop; + + container: + image: "spisarski/siddhi" + imagePullPolicy: Always + +--- +apiVersion: v1 +kind: Service +metadata: + name: trpt-to-kafka-0 +spec: + type: LoadBalancer + clusterIP: {{ ae_ip }} + selector: + siddhi.io/instance: trpt-to-kafka-0 + siddhi.io/name: SiddhiProcess + siddhi.io/part-of: siddhi-operator + siddhi.io/version: 0.2.2 + ports: + - port: 556 + targetPort: 556 + protocol: UDP diff --git a/playbooks/siddhi/setup.yml b/playbooks/siddhi/setup_siddhi_maven.yml similarity index 93% rename from playbooks/siddhi/setup.yml rename to playbooks/siddhi/setup_siddhi_maven.yml index 8be739bc..0871d18d 100644 --- a/playbooks/siddhi/setup.yml +++ b/playbooks/siddhi/setup_siddhi_maven.yml @@ -18,8 +18,8 @@ # ctrl-C will gracefully exit. # mvn exec:java -Dexec.mainClass=io.siddhi.extension.map.p4.StartSiddhiRuntime "-Dexec.args=/home/ubuntu/siddhi-map-p4-trpt/docs/siddhi/examples/convert_trpt.siddhi /home/ubuntu/siddhi-map-p4-trpt/docs/siddhi/examples/simple_ddos_detection.siddhi" -f pom.xml --- -- import_playbook: setup_jdk.yml -- import_playbook: setup_kafka.yml +- import_playbook: ../general/setup_jdk.yml +- import_playbook: ../general/setup_kafka.yml - import_playbook: setup_siddhi_p4.yml - import_playbook: ../general/setup_source.yml - import_playbook: ../general/final_env_setup.yml diff --git a/playbooks/mininet/setup_host.yml b/playbooks/siddhi/setup_siddhi_minikube.yml similarity index 61% rename from playbooks/mininet/setup_host.yml rename to playbooks/siddhi/setup_siddhi_minikube.yml index abd383af..3e0926d0 100644 --- a/playbooks/mininet/setup_host.yml +++ b/playbooks/siddhi/setup_siddhi_minikube.yml @@ -1,4 +1,4 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. +# Copyright (c) 2021 Cable Television Laboratories, Inc. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -10,9 +10,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ---- -- import_playbook: ../general/setup_source.yml - vars: - python_unit_tests: true -- import_playbook: generate_inventory.yml +# Playbook that is responsible for putting together an AWS image for running +# Siddhi with the new "udp" source & P4 Telemetry report mapping extensions +# running in Kubernetes +--- +- import_playbook: ../general/setup_siddhi_operator.yml +- import_playbook: ../general/final_env_setup.yml diff --git a/playbooks/siddhi/templates/convert_trpt.siddhi.j2 b/playbooks/siddhi/templates/convert_trpt.siddhi.j2 index 468b7083..f2a33720 100644 --- a/playbooks/siddhi/templates/convert_trpt.siddhi.j2 +++ b/playbooks/siddhi/templates/convert_trpt.siddhi.j2 @@ -13,7 +13,7 @@ * limitations under the License. */ -@app:name('UDP-Source-TRPT') +@App:name('UDPSourceTRPT') @source(type='udp', listen.port='{{ telem_rpt_port }}', @map(type='p4-trpt', @attributes(in_type='telemRptHdr.inType', full_json='jsonString'))) define stream trptUdpStream (in_type int, full_json object); diff --git a/playbooks/siddhi/templates/minikube_lb.service.j2 b/playbooks/siddhi/templates/minikube_lb.service.j2 new file mode 100644 index 00000000..bc3ba16d --- /dev/null +++ b/playbooks/siddhi/templates/minikube_lb.service.j2 @@ -0,0 +1,14 @@ +[Unit] +Description=Minikube service for TPS AE +Wants=minikube.service +Requisite={{ minikube_srvc }} +After={{ minikube_srvc }} +PartOf={{ minikube_srvc }} + +[Service] +User=ubuntu +#Group=docker +ExecStart=/usr/bin/minikube tunnel +ExecStop=/usr/bin/systemctl stop minikube.service +[Install] +WantedBy=multi-user.target diff --git a/playbooks/siddhi/templates/minikube_service.service.j2 b/playbooks/siddhi/templates/minikube_service.service.j2 new file mode 100644 index 00000000..f09fb2d9 --- /dev/null +++ b/playbooks/siddhi/templates/minikube_service.service.j2 @@ -0,0 +1,16 @@ +[Unit] +Description=Minikube service for TPS AE +Requires=network.target docker.service +After=syslog.target network.target docker.service +PartOf={{ part_of_srvc }} + +[Service] +Type=oneshot +RemainAfterExit=yes +User=ubuntu +Group=docker +ExecStart=/usr/bin/minikube start --memory=4096 +ExecStop=/usr/bin/minikube stop + +[Install] +WantedBy=multi-user.target diff --git a/playbooks/siddhi/templates/simple_ddos_clear_drop.siddhi.j2 b/playbooks/siddhi/templates/simple_ddos_clear_drop.siddhi.j2 index c4117754..eb1a7c6c 100644 --- a/playbooks/siddhi/templates/simple_ddos_clear_drop.siddhi.j2 +++ b/playbooks/siddhi/templates/simple_ddos_clear_drop.siddhi.j2 @@ -13,7 +13,7 @@ * limitations under the License. */ -@app:name('Kafka-Source-Drop-JSON') +@App:name('KafkaSourceDropJSON') @source(type='kafka', topic.list='{{ kafka_trpt_drop_topic }}', bootstrap.servers='{{ kafka_host_port }}', group.id='test', threading.option='single.thread', diff --git a/playbooks/siddhi/templates/simple_ddos_detection.siddhi.j2 b/playbooks/siddhi/templates/simple_ddos_detection.siddhi.j2 index 54b97080..072cbe12 100644 --- a/playbooks/siddhi/templates/simple_ddos_detection.siddhi.j2 +++ b/playbooks/siddhi/templates/simple_ddos_detection.siddhi.j2 @@ -13,7 +13,7 @@ * limitations under the License. */ -@app:name('Kafka-Source-Packet-JSON') +@App:name('KafkaSourcePacketJSON') @source(type='kafka', topic.list='{{ kafka_trpt_pkt_topic }}', bootstrap.servers='{{ kafka_host_port }}', group.id='test', threading.option='single.thread', diff --git a/playbooks/tofino/setup_minikube_siddhi_ae.yml b/playbooks/tofino/setup_minikube_siddhi_ae.yml new file mode 100644 index 00000000..881034d6 --- /dev/null +++ b/playbooks/tofino/setup_minikube_siddhi_ae.yml @@ -0,0 +1,134 @@ +# Copyright (c) 2021 Cable Television Laboratories, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- hosts: "{{ host_val | default('all') }}" + gather_facts: no + tasks: + - name: Create Service File to /etc/systemd/system/minikube.service + become: yes + template: + src: ../siddhi/templates/minikube_service.service.j2 + dest: /etc/systemd/system/minikube.service + vars: + part_of_srvc: tps-tofino-ae.service + + - name: Create Service File to /etc/systemd/system/tps-tofino-ae.service + become: yes + template: + src: ../siddhi/templates/minikube_lb.service.j2 + dest: /etc/systemd/system/tps-tofino-ae.service + vars: + minikube_srvc: minikube.service + + - name: Start TPS AE + become: yes + systemd: + name: tps-tofino-ae.service + state: restarted + enabled: yes + register: minkube_start_rc + retries: 3 + delay: 5 + until: minkube_start_rc is not failed + + - name: Wait for tps-tofino-ae service to be completely up... + command: kubectl get po + register: kube_pods_out + retries: 10 + delay: 5 + until: kube_pods_out.stdout.find("Running") != -1 + + - name: Show pods + debug: + var: kube_pods_out.stdout_lines + + - name: Ensure {{ remote_scripts_dir }} scripts directory has been created + become: yes + file: + path: "{{ remote_scripts_dir }}" + state: directory + + - name: Query the Kafka Service clusterIP + command: kubectl get svc -n kafka my-cluster-kafka-bootstrap -o jsonpath="{.spec.clusterIPs[0]}" + register: cluster_ip_out + + - name: Define the K8s CRDs + set_fact: + the_crds: + - "kafka-trpt-ddos-detection.yaml" + - "kafka-trpt-drop-clear.yaml" + - "trpt-to-kafka.yaml" + kafka_svc_ip: "{{ cluster_ip_out.stdout_lines[0] }}" + + - debug: + var: the_crds + + - name: Applying J2 templates of the siddhi K8s CRDs to {{ remote_scripts_dir }} with Kafka service IP - {{ kafka_svc_ip }} + become: yes + template: + src: "{{ trans_sec_dir }}/playbooks/siddhi/kubernetes/templates/{{ item }}.j2" + dest: "{{ remote_scripts_dir }}/{{ item }}" + loop: "{{ the_crds }}" + + - name: Install siddhi K8s SiddhiProcess + command: "kubectl apply -f {{ remote_scripts_dir }}/{{ item }}" + loop: "{{ the_crds }}" + + - name: Wait for K8s deployment/trpt-to-kafka-0 to become available + command: kubectl wait --for=condition=available --timeout=600s deployment/trpt-to-kafka-0 -n default + register: trpt_to_kafka_check + retries: 10 + delay: 20 + until: trpt_to_kafka_check is not failed + + - name: Wait for K8s deployment/kafka-trpt-ddos-detect-0 to become available + command: kubectl wait --for=condition=available --timeout=600s deployment/kafka-trpt-ddos-detect-0 -n default + register: ddos_check + retries: 5 + delay: 10 + until: ddos_check is not failed + + - name: Wait for K8s deployment/kafka-trpt-drop-clear-0 to become available + command: kubectl wait --for=condition=available --timeout=600s deployment/kafka-trpt-drop-clear-0 -n default + register: drop_check + retries: 5 + delay: 10 + until: drop_check is not failed + + - name: Get pods in default namespace + command: kubectl get pods -n default + register: kube_pods_out + + - name: Show pods + debug: + var: kube_pods_out.stdout_lines + + # TODO - Find a better means to obtain this bridge as things may break as soon as another interface prefaced with "br-" is created + - name: Query for the docker/K8s bridge + shell: "ip -o link show | awk -F': ' '{print $2}'| grep br-" + register: bridge_name_out + + - name: Store the CRDs + set_fact: + docker_bridge_name: "{{ bridge_name_out.stdout_lines[0] }}" + + # TODO - Reconsider this approach which was hacked in place as TRPTs aren't getting routed into the pod + - name: Forward all {{ clone_tun_name }} packets to the Docker/K8s service bridge {{ docker_bridge_name }} + command: "sudo iptables -A FORWARD -i {{ clone_tun_name }} -o {{ docker_bridge_name }} -j ACCEPT" + + - name: Stop tps-tofino-ae.service + become: yes + systemd: + name: tps-tofino-ae.service + state: stopped diff --git a/playbooks/tofino/setup_nodes-full.yml b/playbooks/tofino/setup_nodes-full.yml deleted file mode 100644 index 4a8b385f..00000000 --- a/playbooks/tofino/setup_nodes-full.yml +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -- import_playbook: ../general/setup_source.yml - vars: - host_val: - - hosts - trans_sec_source_dir: "{{ orch_trans_sec_dir }}" - python_unit_tests: false - -# Create virtual interfaces on switches -- import_playbook: setup_virt_eth.yml - vars: - host_val: switches - remote_sde_dir: "{{ sde_dir }}" - -# Setup GRE Tunnels -- import_playbook: tunnel/setup_gre_tunnels.yml - -# TODO/FIXME - Call setup_tofino_switch playbook so it can run in parallel - -# Start tofino-model and tofino-switchd on the core switch -- import_playbook: setup_tofino_switch.yml - vars: - switches: - - core - host_val: core - p4_prog: core.p4 - topo_dict: "{{ lookup('file','{{ topo_file_loc }}') | from_yaml }}" - tunnels: "{{ topo_dict.switches['core'].tunnels }}" - load_tofino_p4rt: False - -# Start tofino-model and tofino-switchd on the aggregate switch -- import_playbook: setup_tofino_switch.yml - vars: - switches: - - aggregate - host_val: aggregate - p4_prog: aggregate.p4 - topo_dict: "{{ lookup('file','{{ topo_file_loc }}') | from_yaml }}" - tunnels: "{{ topo_dict.switches['aggregate'].tunnels }}" - -# Start tofino-model and tofino-switchd on gateway1 -- import_playbook: setup_tofino_switch.yml - vars: - switches: - - gateway1 - - gateway2 - - gateway3 - p4_prog: gateway.p4 - -# Start SDN Controller -- import_playbook: ../general/start_service.yml - vars: - host_val: controller - service_name: tps-tofino-sdn - scenario_name: full - local_srvc_script_tmplt_file: "{{ orch_trans_sec_dir }}/playbooks/general/templates/sdn_controller.sh.j2" - port_to_wait: "{{ sdn_port }}" - wait_timeout: 30 - load_p4: False - -# Start AE -- import_playbook: ../general/start_service.yml - vars: - host_val: ae - service_name: tps-tofino-ae - local_srvc_script_tmplt_file: "{{ orch_trans_sec_dir }}/playbooks/general/templates/ae_service.sh.j2" - srvc_type: SIMPLE - sdn_url: "http://{{ sdn_ip }}:{{ sdn_port }}" - monitor_intf: "{{ ae_monitor_intf }}" diff --git a/playbooks/tofino/setup_nodes-lab_trial.yml b/playbooks/tofino/setup_nodes-lab_trial.yml index 1216ef11..539cc9be 100644 --- a/playbooks/tofino/setup_nodes-lab_trial.yml +++ b/playbooks/tofino/setup_nodes-lab_trial.yml @@ -11,6 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +--- # Copy TPS source from orchestrator to all hosts - import_playbook: ../general/setup_source.yml vars: @@ -19,6 +20,12 @@ trans_sec_source_dir: "{{ orch_trans_sec_dir }}" python_unit_tests: false +# Setup AE +- import_playbook: setup_minikube_siddhi_ae.yml + vars: + host_val: ae + clone_tun_name: core-tun1 + # Create virtual interfaces on switches - import_playbook: setup_virt_eth.yml vars: @@ -46,25 +53,3 @@ wait_timeout: 60 load_p4: False -# Start AE -- import_playbook: ../general/start_service.yml - vars: - host_val: ae - service_name: tps-tofino-ae - srvc_start_pause_time: 45 - local_srvc_script_tmplt_file: "{{ orch_trans_sec_dir }}/playbooks/general/templates/siddhi_p4_service.sh.j2" - srvc_type: SIMPLE - sdn_url: "http://{{ sdn_ip }}:{{ sdn_port }}" - telem_rpt_port: 556 - kafka_host_port: localhost:9092 - kafka_trpt_pkt_topic: trptPacket - kafka_trpt_drop_topic: trptDrop - alert_pkt_count: 10 - alert_window_secs: 1 - templates: - - src: "{{ orch_trans_sec_dir }}/playbooks/siddhi/templates/convert_trpt.siddhi.j2" - dest: "{{ remote_scripts_dir }}/convert_trpt.siddhi" - - src: "{{ orch_trans_sec_dir }}/playbooks/siddhi/templates/simple_ddos_detection.siddhi.j2" - dest: "{{ remote_scripts_dir }}/simple_ddos_detection.siddhi" - - src: "{{ orch_trans_sec_dir }}/playbooks/siddhi/templates/simple_ddos_clear_drop.siddhi.j2" - dest: "{{ remote_scripts_dir }}/simple_ddos_clear.siddhi" diff --git a/playbooks/tofino/setup_orchestrator-full.yml b/playbooks/tofino/setup_orchestrator-full.yml deleted file mode 100644 index e09d1215..00000000 --- a/playbooks/tofino/setup_orchestrator-full.yml +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright (c) 2020 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -- import_playbook: setup_orchestrator.yml - vars: - topology_template: templates/topology_template-full.yaml.j2 diff --git a/playbooks/tofino/setup_orchestrator.yml b/playbooks/tofino/setup_orchestrator.yml index 5ccbc7aa..96d327e7 100644 --- a/playbooks/tofino/setup_orchestrator.yml +++ b/playbooks/tofino/setup_orchestrator.yml @@ -20,6 +20,12 @@ vars: local_topology: /tmp/tofino-topology.yaml tasks: + - name: Create {{ remote_scripts_dir }} + become: yes + file: + path: "{{ remote_scripts_dir }}" + state: directory + - name: Apply template to topology configuration file {{ topology_template }} become: yes template: diff --git a/playbooks/tofino/setup_tofino_switch.yml b/playbooks/tofino/setup_tofino_switch.yml index 669cf892..efd3dd5d 100644 --- a/playbooks/tofino/setup_tofino_switch.yml +++ b/playbooks/tofino/setup_tofino_switch.yml @@ -27,6 +27,7 @@ # Start tofino model chip emulator - import_playbook: ../general/start_service.yml vars: + setup_for_hw: "{{ from_hw | default(False) }}" host_val: "{{ switches }}" service_name: tps-tofino-model topo_dict: "{{ lookup('file','{{ topo_file_loc }}') | from_yaml }}" @@ -37,7 +38,7 @@ prog_name: "{{ p4_prog }}" additional_tmplt_file: "{{ trans_sec_dir }}/playbooks/tofino/templates/tofino-model-veth-port-mapping.json.j2" additional_tmplt_out_file: "{{ remote_scripts_dir }}/port-mapping.json" - when: not from_hw | bool + when: not setup_for_hw | bool # Start switchd - import_playbook: ../general/start_service.yml diff --git a/playbooks/tofino/setup_virt_eth.yml b/playbooks/tofino/setup_virt_eth.yml index a318d7ab..363428dc 100644 --- a/playbooks/tofino/setup_virt_eth.yml +++ b/playbooks/tofino/setup_virt_eth.yml @@ -20,10 +20,11 @@ SDE: "{{ remote_sde_dir }}" SDE_INSTALL: "{{ remote_sde_dir }}/install" vars: - setup_veth: "{{ not from_hw|bool | default(True) }}" + setup_for_hw: "{{ from_hw | default(False) }}" + setup_veth: "{{ not setup_for_hw | bool }}" tasks: - debug: - var: from_hw + var: setup_for_hw - debug: var: setup_veth - name: Setup DMA diff --git a/playbooks/tofino/templates/topology_template-lab_trial.yaml.j2 b/playbooks/tofino/templates/topology_template-lab_trial.yaml.j2 index a0c356a4..be25796e 100644 --- a/playbooks/tofino/templates/topology_template-lab_trial.yaml.j2 +++ b/playbooks/tofino/templates/topology_template-lab_trial.yaml.j2 @@ -73,7 +73,8 @@ hosts: {% if host_user is defined %} user: {{ host_user }} {% endif %} - public_ip: {{ ae_ip }} + public_ip: {{ ae_mgmt_ip }} + streaming_ip: {{ ae_ip }} tun1_ip: {{ ae_tun1_ip }} tun1_mac: {{ ae_tun1_mac }} tunnels: diff --git a/playbooks/tofino/tunnel/setup_gre_tunnels.yml b/playbooks/tofino/tunnel/setup_gre_tunnels.yml index b1190e6a..e326e644 100644 --- a/playbooks/tofino/tunnel/setup_gre_tunnels.yml +++ b/playbooks/tofino/tunnel/setup_gre_tunnels.yml @@ -19,10 +19,11 @@ gather_facts: yes become: yes vars: - setup_tunnels: "{{ not from_hw|bool | default(True) }}" + setup_for_hw: "{{ from_hw | default(False) }}" + setup_tunnels: "{{ not setup_for_hw | bool }}" tasks: - debug: - var: from_hw + var: setup_for_hw - debug: var: setup_tunnels - block: @@ -38,7 +39,7 @@ name: ip_gre state: present - - name: install ip_gre module + - name: Setup ipv4 forwarding lineinfile: path: /etc/sysctl.conf line: "net.ipv4.ip_forward=1" diff --git a/tests/trans_sec/analytics/__init__.py b/tests/trans_sec/analytics/__init__.py deleted file mode 100644 index e9b9345f..00000000 --- a/tests/trans_sec/analytics/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Unit tests for convert.py diff --git a/tests/trans_sec/analytics/oinc_tests.py b/tests/trans_sec/analytics/oinc_tests.py deleted file mode 100644 index 0269450f..00000000 --- a/tests/trans_sec/analytics/oinc_tests.py +++ /dev/null @@ -1,730 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Unit tests for http_session.py -import logging -import sys -import time -import unittest -from random import randrange, randint - -import ipaddress -import mock -from scapy.layers.inet import IP, UDP, TCP -from scapy.layers.inet6 import IPv6 -from scapy.layers.l2 import Ether - -from trans_sec import consts -from trans_sec.analytics import oinc -from trans_sec.analytics.oinc import SimpleAE -from trans_sec.packet.inspect_layer import ( - IntShim, IntMeta2, IntHeader, SourceIntMeta, IntMeta1, UdpInt, - TelemetryReport, DropReport) -from trans_sec.utils import tps_utils -from trans_sec.utils.http_session import HttpSession - -logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) - -logger = logging.getLogger('oinc_tests') - - -class SimpleAETests(unittest.TestCase): - """ - Unit tests for the class SimpleAE - """ - - def setUp(self): - self.ae = SimpleAE(mock.Mock(HttpSession), packet_count=20, - sample_interval=2) - self.sport = randrange(1000, 8000) - self.dport = randrange(1000, 8000) - self.ae_ip = '192.168.1.2' - self.dst_ipv4 = '10.1.0.2' - self.src_ipv4 = '10.1.0.6' - self.dst_ipv6 = ipaddress.ip_address( - '0000:0000:0000:0000:0000:0001:0000:0001') - self.src_ipv4 = '10.2.0.1' - self.src_ipv6 = ipaddress.ip_address( - '0000:0000:0000:0000:0000:0002:0000:0001') - self.dst_mac = rand_mac() - self.src_mac = rand_mac() - # self.orig_mac = rand_mac() - self.orig_mac = '00:00:00:02:02:00' - logger.info('Test sport - [%s] dport - [%s]', self.sport, self.dport) - - self.int_pkt_ipv4_udp = ( - Ether(src='00:00:00:00:01:01', dst=self.dst_mac) / - IP(dst=self.dst_ipv4, src=self.src_ipv4, - proto=consts.UDP_PROTO) / - UdpInt(dport=consts.UDP_INT_DST_PORT) / - IntShim(length=9, next_proto=consts.UDP_PROTO) / - IntHeader(meta_len=1) / - IntMeta1(switch_id=3) / - IntMeta2(switch_id=2) / - SourceIntMeta(switch_id=1, orig_mac=self.orig_mac) / - UDP(dport=self.dport, sport=self.sport) / - 'hello transparent-security' - ) - - self.ipv4_hash = tps_utils.create_attack_hash( - mac=self.orig_mac, port=self.dport, ip_addr=self.dst_ipv4, - ipv6_addr='::') - ip_len = (consts.IPV4_HDR_LEN + consts.UDP_INT_HDR_LEN - + consts.DRPT_LEN + consts.UDP_HDR_LEN - + consts.DRPT_PAYLOAD_LEN) - udp_int_len = ip_len - consts.IPV4_HDR_LEN - - self.int_drop_rpt_ipv4_udp = ( - Ether(type=consts.IPV4_TYPE) / - IP(dst=self.ae_ip, src=self.src_ipv4, len=ip_len, - proto=consts.UDP_PROTO) / - UdpInt(sport=consts.UDP_INT_SRC_PORT, - dport=consts.UDP_TRPT_DST_PORT, - len=udp_int_len) / - DropReport( - ver=consts.DRPT_VER, - node_id=0, - in_type=consts.DRPT_IN_TYPE, - rpt_len=consts.DRPT_REP_LEN, md_len=consts.DRPT_MD_LEN, - rep_md_bits=consts.DRPT_MD_BITS, - domain_id=consts.TRPT_DOMAIN_ID, - var_opt_bsmd=consts.DRPT_BS_MD, - timestamp=int(time.time()), - drop_count=5, - drop_hash=self.ipv4_hash) - ) - - self.int_pkt_ipv4_tcp = ( - Ether(src='00:00:00:00:01:01', dst=self.dst_mac) / - IP(dst=self.dst_ipv4, src=self.src_ipv4, - proto=consts.UDP_PROTO) / - UdpInt(dport=consts.UDP_INT_DST_PORT) / - IntShim(length=9, next_proto=consts.TCP_PROTO) / - IntHeader(meta_len=1) / - IntMeta1(switch_id=3) / - IntMeta2(switch_id=2) / - SourceIntMeta(switch_id=1, orig_mac=self.orig_mac) / - TCP(dport=self.dport, sport=self.sport) / - 'hello transparent-security' - ) - - self.int_pkt_ipv6_udp = ( - Ether(src='00:00:00:00:01:01', dst=self.dst_mac, - type=consts.IPV6_TYPE) / - IPv6(dst=self.dst_ipv6, - src=self.src_ipv6, - nh=consts.UDP_PROTO) / - UDP(dport=consts.UDP_INT_DST_PORT) / - IntShim(length=9, next_proto=consts.UDP_PROTO) / - IntHeader(meta_len=1) / - IntMeta1(switch_id=3) / - IntMeta2(switch_id=2) / - SourceIntMeta(switch_id=1, orig_mac=self.orig_mac) / - UDP(dport=self.dport, sport=self.sport) / - 'hello transparent-security' - ) - - self.int_pkt_ipv6_tcp = ( - Ether(src='00:00:00:00:01:01', dst=self.dst_mac, - type=consts.IPV6_TYPE) / - IPv6(dst=self.dst_ipv6, - src=self.src_ipv6, - nh=consts.UDP_PROTO) / - UDP(dport=consts.UDP_INT_DST_PORT) / - IntShim(length=9, next_proto=consts.TCP_PROTO) / - IntHeader(meta_len=1) / - IntMeta1(switch_id=3) / - IntMeta2(switch_id=2) / - SourceIntMeta(switch_id=1, orig_mac=self.orig_mac) / - TCP(dport=self.dport, sport=self.sport) / - 'hello transparent-security' - ) - - self.trpt_pkt_ipv4_out_ipv4_in_udp = ( - Ether(src=self.src_mac, dst=self.dst_mac) / - IP(dst=self.dst_ipv4, src=self.src_ipv4, - proto=consts.UDP_PROTO) / - UDP(sport=0, dport=consts.UDP_TRPT_DST_PORT, - # udp + telemetry header size - len=len(self.int_pkt_ipv4_udp) + 20 + 20) / - TelemetryReport(domain_id=consts.TRPT_DOMAIN_ID) / - self.int_pkt_ipv4_udp - ) - - self.trpt_pkt_ipv4_out_ipv6_in_udp = ( - Ether(src=self.src_mac, dst=self.dst_mac) / - IP(dst=self.dst_ipv4, src=self.src_ipv4, - proto=consts.UDP_PROTO) / - UDP(sport=0, dport=consts.UDP_TRPT_DST_PORT, - # udp + telemetry header size - len=len(self.int_pkt_ipv4_udp) + 20 + 20) / - TelemetryReport(domain_id=consts.TRPT_DOMAIN_ID) / - self.int_pkt_ipv6_udp - ) - - self.trpt_pkt_ipv4_out_ipv4_in_tcp = ( - Ether(src=self.src_mac, dst=self.dst_mac) / - IP(dst=self.dst_ipv4, src=self.src_ipv4, - proto=consts.UDP_PROTO) / - UDP(sport=0, dport=consts.UDP_TRPT_DST_PORT, - # udp + telemetry header size - len=len(self.int_pkt_ipv4_udp) + 20 + 20) / - TelemetryReport(domain_id=consts.TRPT_DOMAIN_ID) / - self.int_pkt_ipv4_tcp - ) - - self.trpt_pkt_ipv4_out_ipv6_in_tcp = ( - Ether(src=self.src_mac, dst=self.dst_mac) / - IP(dst=self.dst_ipv4, src=self.src_ipv4, - proto=consts.UDP_PROTO) / - UDP(sport=0, dport=consts.UDP_TRPT_DST_PORT, - # udp + telemetry header size - len=len(self.int_pkt_ipv4_udp) + 20 + 20) / - TelemetryReport(domain_id=consts.TRPT_DOMAIN_ID) / - self.int_pkt_ipv6_tcp - ) - - def test_extract_ipv4_udp_packet(self): - """ - Tests to ensure that an IPv4 UDP single packet will be parsed properly - """ - int_data = oinc.extract_int_data(self.int_pkt_ipv4_udp[Ether]) - self.assertEqual(self.orig_mac, int_data['devMac']) - self.assertEqual(self.src_ipv4, int_data['devAddr']) - self.assertEqual(self.dst_ipv4, int_data['dstAddr']) - self.assertEqual(self.dport, int_data['dstPort']) - self.assertEqual(consts.UDP_PROTO, int_data['protocol']) - - def test_extract_ipv4_udp_packet_trpt(self): - """ - Tests to ensure that an IPv4 UDP single packet will be parsed properly - """ - int_data = oinc.extract_trpt_data( - self.trpt_pkt_ipv4_out_ipv4_in_udp[UDP]) - self.assertEqual(self.orig_mac, int_data['devMac']) - self.assertEqual(self.src_ipv4, int_data['devAddr']) - self.assertEqual(self.dst_ipv4, int_data['dstAddr']) - self.assertEqual(self.dport, int_data['dstPort']) - self.assertEqual(consts.UDP_PROTO, int_data['protocol']) - - def test_extract_ipv4_udp_packet_drpt(self): - """ - Tests to ensure that an IPv4 UDP drop report will be parsed properly - """ - hash_key, count = oinc.extract_drop_rpt( - self.int_drop_rpt_ipv4_udp[UdpInt]) - self.assertEqual(self.ipv4_hash, hash_key) - self.assertEqual(5, count) - - def test_extract_ipv4_tcp_packet(self): - """ - Tests to ensure that an IPv4 UDP single packet will be parsed properly - """ - int_data = oinc.extract_int_data(self.int_pkt_ipv4_tcp[Ether]) - self.assertEqual(self.orig_mac, int_data['devMac']) - self.assertEqual(self.src_ipv4, int_data['devAddr']) - self.assertEqual(self.dst_ipv4, int_data['dstAddr']) - self.assertEqual(self.dport, int_data['dstPort']) - self.assertEqual(consts.TCP_PROTO, int_data['protocol']) - - def test_extract_ipv4_tcp_packet_trpt(self): - """ - Tests to ensure that an IPv4 UDP single packet will be parsed properly - """ - logger.debug('Packet to test - [%s]', - self.trpt_pkt_ipv4_out_ipv4_in_tcp) - int_data = oinc.extract_trpt_data( - self.trpt_pkt_ipv4_out_ipv4_in_tcp[UDP]) - self.assertEqual(self.orig_mac, int_data['devMac']) - self.assertEqual(self.src_ipv4, int_data['devAddr']) - self.assertEqual(self.dst_ipv4, int_data['dstAddr']) - self.assertEqual(self.dport, int_data['dstPort']) - self.assertEqual(consts.TCP_PROTO, int_data['protocol']) - - def test_extract_ipv6_udp_packet(self): - """ - Tests to ensure that an IPv4 UDP single packet will be parsed properly - """ - int_data = oinc.extract_int_data(self.int_pkt_ipv6_udp[Ether]) - self.assertEqual(self.orig_mac, int_data['devMac']) - self.assertEqual(str(self.src_ipv6), int_data['devAddr']) - self.assertEqual(str(self.dst_ipv6), int_data['dstAddr']) - self.assertEqual(self.dport, int_data['dstPort']) - self.assertEqual(consts.UDP_PROTO, int_data['protocol']) - - def test_extract_ipv6_udp_packet_trpt(self): - """ - Tests to ensure that an IPv6 UDP single packet will be parsed properly - """ - int_data = oinc.extract_trpt_data( - self.trpt_pkt_ipv4_out_ipv6_in_udp[UDP]) - self.assertEqual(self.orig_mac, int_data['devMac']) - self.assertEqual(str(self.src_ipv6), int_data['devAddr']) - self.assertEqual(str(self.dst_ipv6), int_data['dstAddr']) - self.assertEqual(self.dport, int_data['dstPort']) - self.assertEqual(consts.UDP_PROTO, int_data['protocol']) - - def test_extract_ipv6_tcp_packet(self): - """ - Tests to ensure that an IPv6 TCP single packet will be parsed properly - """ - int_data = oinc.extract_int_data(self.int_pkt_ipv6_tcp[Ether]) - self.assertEqual(self.orig_mac, int_data['devMac']) - self.assertEqual(str(self.src_ipv6), int_data['devAddr']) - self.assertEqual(str(self.dst_ipv6), int_data['dstAddr']) - self.assertEqual(self.dport, int_data['dstPort']) - self.assertEqual(consts.TCP_PROTO, int_data['protocol']) - - def test_extract_ipv6_tcp_packet_trpt(self): - """ - Tests to ensure that an IPv6 TCP single packet will be parsed properly - """ - int_data = oinc.extract_trpt_data( - self.trpt_pkt_ipv4_out_ipv6_in_tcp[UDP]) - self.assertEqual(self.orig_mac, int_data['devMac']) - self.assertEqual(str(self.src_ipv6), int_data['devAddr']) - self.assertEqual(str(self.dst_ipv6), int_data['dstAddr']) - self.assertEqual(self.dport, int_data['dstPort']) - self.assertEqual(consts.TCP_PROTO, int_data['protocol']) - - def test_process_single_drop_rpt_packet(self): - """ - Tests to ensure that an IPv4 UDP single packet is handled without Error - note: only testing via the handle_packet() API which would be called by - by the scapy sniffer thread - :return: - """ - self.assertFalse(self.ae.process_drop_rpt(self.int_pkt_ipv4_udp)) - - def test_process_single_ipv4_udp_packet(self): - """ - Tests to ensure that an IPv4 UDP single packet is handled without Error - note: only testing via the handle_packet() API which would be called by - by the scapy sniffer thread - :return: - """ - self.assertFalse(self.ae.process_packet(self.int_pkt_ipv4_udp)) - - def test_process_single_ipv4_udp_packet_trpt(self): - """ - Tests to ensure that an IPv4 UDP single packet is handled without Error - note: only testing via the handle_packet() API which would be called by - by the scapy sniffer thread - :return: - """ - self.assertFalse( - self.ae.process_packet(self.trpt_pkt_ipv4_out_ipv4_in_udp)) - - def test_process_single_ipv6_udp_packet(self): - """ - Tests to ensure that an IPv6 UDP single packet is handled without Error - note: only testing via the handle_packet() API which would be called by - by the scapy sniffer thread - :return: - """ - self.ae.process_packet(self.int_pkt_ipv6_udp) - - def test_process_single_ipv6_udp_packet_trpt(self): - """ - Tests to ensure that an IPv6 UDP single packet is handled without Error - note: only testing via the handle_packet() API which would be called by - by the scapy sniffer thread - :return: - """ - self.ae.process_packet( - self.trpt_pkt_ipv4_out_ipv6_in_udp, consts.UDP_TRPT_DST_PORT) - - def test_process_single_ipv4_tcp_packet(self): - """ - Tests to ensure that a single IPv4 TCP packet is handled without Error - note: only testing via the handle_packet() API which would be called by - by the scapy sniffer thread - :return: - """ - self.ae.process_packet(self.int_pkt_ipv4_tcp) - - def test_process_single_ipv4_tcp_packet_trpt(self): - """ - Tests to ensure that a single IPv4 TCP packet is handled without Error - note: only testing via the handle_packet() API which would be called by - by the scapy sniffer thread - :return: - """ - self.ae.process_packet( - self.trpt_pkt_ipv4_out_ipv4_in_tcp, consts.UDP_TRPT_DST_PORT) - - def test_process_single_ipv6_tcp_packet(self): - """ - Tests to ensure that a single IPv6 TCP packet is handled without Error - note: only testing via the handle_packet() API which would be called by - by the scapy sniffer thread - :return: - """ - self.ae.process_packet(self.int_pkt_ipv6_tcp) - - def test_process_single_ipv6_tcp_packet_trpt(self): - """ - Tests to ensure that a single IPv6 TCP packet is handled without Error - note: only testing via the handle_packet() API which would be called by - by the scapy sniffer thread - :return: - """ - self.ae.process_packet( - self.trpt_pkt_ipv4_out_ipv6_in_tcp, consts.UDP_TRPT_DST_PORT) - - def test_start_one_ipv4_udp_attack(self): - """ - Tests to ensure that one IPv4 UDP attack has been triggered - :return: - """ - for index in range(0, self.ae.packet_count + 1): - logger.debug('Processing packet #%s', index) - ret_val = self.ae.process_packet(self.int_pkt_ipv4_udp) - if index < self.ae.packet_count: - self.assertFalse(ret_val) - else: - self.assertTrue(ret_val) - - def test_start_one_ipv4_udp_attack_trpt(self): - """ - Tests to ensure that one IPv4 UDP attack has been triggered - :return: - """ - for index in range(0, self.ae.packet_count + 1): - logger.debug('Processing packet #%s', index) - ret_val = self.ae.process_packet( - self.trpt_pkt_ipv4_out_ipv4_in_udp, consts.UDP_TRPT_DST_PORT) - if index < self.ae.packet_count: - self.assertFalse(ret_val) - else: - self.assertTrue(ret_val) - - def test_ipv4_udp_proc_attack_and_drop(self): - """ - Tests to ensure that one IPv4 UDP attack has been triggered and has - released after processing several drop reports - :return: - """ - ret_val = False - for i in range(0, self.ae.packet_count + 1): - logger.debug('Processing packet #%s', i) - ret_val = self.ae.process_packet( - self.trpt_pkt_ipv4_out_ipv4_in_udp, consts.UDP_TRPT_DST_PORT) - if i < self.ae.packet_count: - self.assertFalse(ret_val) - else: - self.assertTrue(ret_val) - - # Ensure didn't loop again with False - self.assertTrue(ret_val) - - drop_ret = False - for j in range(0, 4): - drop_ret = self.ae.process_drop_rpt(self.int_drop_rpt_ipv4_udp) - if j < 3: - self.assertFalse(drop_ret) - else: - self.assertTrue(drop_ret) - - # Ensure didn't loop again with False - self.assertTrue(drop_ret) - - proc_ret = self.ae.process_packet( - self.trpt_pkt_ipv4_out_ipv4_in_udp, consts.UDP_TRPT_DST_PORT) - self.assertFalse(proc_ret) - - def test_start_one_ipv6_udp_attack(self): - """ - Tests to ensure that one IPv6 UDP attack has been triggered - :return: - """ - ret_val = False - for index in range(0, self.ae.packet_count + 1): - logger.debug('Processing packet #%s', index) - ret_val = self.ae.process_packet(self.int_pkt_ipv6_udp) - if index < self.ae.packet_count: - self.assertFalse(ret_val) - else: - self.assertTrue(ret_val) - self.assertTrue(ret_val) - - def test_start_one_ipv6_udp_attack_trpt(self): - """ - Tests to ensure that one IPv6 UDP attack has been triggered - :return: - """ - ret_val = False - for index in range(0, self.ae.packet_count + 1): - logger.debug('Processing packet #%s', index) - ret_val = self.ae.process_packet( - self.trpt_pkt_ipv4_out_ipv6_in_udp, consts.UDP_TRPT_DST_PORT) - if index < self.ae.packet_count: - self.assertFalse(ret_val) - else: - self.assertTrue(ret_val) - self.assertTrue(ret_val) - - def test_start_one_ipv4_tcp_attack(self): - """ - Tests to ensure that one IPv4 TCP attack has been triggered - :return: - """ - for index in range(0, self.ae.packet_count + 1): - logger.debug('Processing packet #%s', index) - ret_val = self.ae.process_packet(self.int_pkt_ipv4_tcp) - if index < self.ae.packet_count: - self.assertFalse(ret_val) - else: - self.assertTrue(ret_val) - - def test_start_one_ipv4_tcp_attack_trpt(self): - """ - Tests to ensure that one IPv4 TCP attack has been triggered - :return: - """ - ret_val = False - for index in range(0, self.ae.packet_count + 1): - logger.debug('Processing packet #%s', index) - ret_val = self.ae.process_packet( - self.trpt_pkt_ipv4_out_ipv4_in_tcp, consts.UDP_TRPT_DST_PORT) - if index < self.ae.packet_count: - self.assertFalse(ret_val) - else: - self.assertTrue(ret_val) - - self.assertTrue(ret_val) - - def test_start_one_ipv6_tcp_attack(self): - """ - Tests to ensure that one IPv6 TCP attack has been triggered - :return: - """ - ret_val = False - for index in range(0, self.ae.packet_count + 1): - logger.debug('Processing packet #%s', index) - ret_val = self.ae.process_packet(self.int_pkt_ipv6_tcp) - if index < self.ae.packet_count: - self.assertFalse(ret_val) - else: - self.assertTrue(ret_val) - self.assertTrue(ret_val) - - def test_start_one_ipv6_tcp_attack_trpt(self): - """ - Tests to ensure that one IPv6 TCP attack has been triggered - :return: - """ - ret_val = False - for index in range(0, self.ae.packet_count + 1): - logger.debug('Processing packet #%s', index) - ret_val = self.ae.process_packet( - self.trpt_pkt_ipv4_out_ipv6_in_tcp, consts.UDP_TRPT_DST_PORT) - if index < self.ae.packet_count: - self.assertFalse(ret_val) - else: - self.assertTrue(ret_val) - self.assertTrue(ret_val) - - def test_start_two_ipv4_udp_attacks(self): - """ - Tests to ensure that two IPv4 UDP attacks have been triggered - :return: - """ - pkt1 = (Ether(src='00:00:00:00:01:01', dst=self.dst_mac) / - IP(dst=self.dst_ipv4, src=self.src_ipv4, - proto=consts.UDP_PROTO) / - UdpInt() / - IntShim(length=9, next_proto=consts.UDP_PROTO) / - IntHeader(meta_len=1) / - IntMeta1(switch_id=3) / - IntMeta2(switch_id=2) / - SourceIntMeta(switch_id=1, orig_mac=self.orig_mac) / - UDP(dport=self.dport, sport=self.sport) / - 'hello transparent-security') - - pkt2 = (Ether(src='00:00:00:00:01:01', dst=self.dst_mac) / - IP(dst=self.dst_ipv4, src=self.src_ipv4, - proto=consts.UDP_PROTO) / - UdpInt() / - IntShim(length=9, next_proto=consts.UDP_PROTO) / - IntHeader(meta_len=1) / - IntMeta1(switch_id=3) / - IntMeta2(switch_id=2) / - SourceIntMeta(switch_id=1, orig_mac=self.orig_mac) / - UDP(dport=self.dport, sport=self.sport) / - 'hello transparent-security') - - for index in range(0, self.ae.packet_count): - logger.info('Iteration #%s', index) - ret_val1 = self.ae.process_packet(pkt1) - ret_val2 = self.ae.process_packet(pkt2) - logger.info('Checking index - [%s] - count - [%s]', - index, self.ae.packet_count) - if index * 2 < self.ae.packet_count: - logger.info('Expecting false - [%s]', ret_val1) - self.assertFalse(ret_val1) - self.assertFalse(ret_val2) - else: - logger.info('Expecting true - [%s]', ret_val1) - self.assertTrue(ret_val1) - self.assertTrue(ret_val2) - - def test_start_two_ipv6_udp_attacks(self): - """ - Tests to ensure that two IPv6 UDP attacks have been triggered - :return: - """ - pkt1 = (Ether(src='00:00:00:00:01:01', dst=self.dst_mac, - type=consts.IPV6_TYPE) / - IPv6(dst=self.dst_ipv6, - src=self.src_ipv6, - nh=consts.UDP_PROTO) / - UdpInt() / - IntShim(length=9, next_proto=consts.UDP_PROTO) / - IntHeader(meta_len=1) / - IntMeta1(switch_id=3) / - IntMeta2(switch_id=2) / - SourceIntMeta(switch_id=1, orig_mac=self.orig_mac) / - UDP(dport=self.dport, sport=self.sport) / - 'hello transparent-security') - - pkt2 = (Ether(src='00:00:00:00:01:01', dst=self.dst_mac, - type=consts.IPV6_TYPE) / - IPv6(dst=self.dst_ipv6, - src=self.src_ipv6, - nh=consts.UDP_PROTO) / - UdpInt() / - IntShim(length=9, next_proto=consts.UDP_PROTO) / - IntHeader(meta_len=1) / - IntMeta1(switch_id=3) / - IntMeta2(switch_id=2) / - SourceIntMeta(switch_id=1, orig_mac=self.orig_mac) / - UDP(dport=self.dport, sport=self.sport) / - 'hello transparent-security') - - for index in range(0, self.ae.packet_count): - logger.info('Iteration #%s', index) - ret_val1 = self.ae.process_packet(pkt1) - ret_val2 = self.ae.process_packet(pkt2) - logger.info('Checking index - [%s] - count - [%s]', - index, self.ae.packet_count) - if index * 2 < self.ae.packet_count: - logger.info('Expecting false - [%s]', ret_val1) - self.assertFalse(ret_val1) - self.assertFalse(ret_val2) - else: - logger.info('Expecting true - [%s]', ret_val1) - self.assertTrue(ret_val1) - self.assertTrue(ret_val2) - - def test_start_two_ipv4_tcp_attacks(self): - """ - Tests to ensure that two IPv4 UDP attacks have been triggered - :return: - """ - pkt1 = (Ether(src='00:00:00:00:01:01', dst=self.dst_mac) / - IP(dst=self.dst_ipv4, src=self.src_ipv4, - proto=consts.UDP_PROTO) / - UdpInt() / - IntShim(length=9, next_proto=consts.TCP_PROTO) / - IntHeader(meta_len=1) / - IntMeta1(switch_id=3) / - IntMeta2(switch_id=2) / - SourceIntMeta(switch_id=1, orig_mac=self.orig_mac) / - TCP(dport=self.dport, sport=self.sport) / - 'hello transparent-security') - - pkt2 = (Ether(src='00:00:00:00:01:01', dst=self.dst_mac) / - IP(dst=self.dst_ipv4, src=self.src_ipv4, - proto=consts.UDP_PROTO) / - UdpInt() / - IntShim(length=9, next_proto=consts.TCP_PROTO) / - IntHeader(meta_len=1) / - IntMeta1(switch_id=3) / - IntMeta2(switch_id=2) / - SourceIntMeta(switch_id=1, orig_mac=self.orig_mac) / - TCP(dport=self.dport, sport=self.sport) / - 'hello transparent-security') - - for index in range(0, self.ae.packet_count): - logger.info('Iteration #%s', index) - ret_val1 = self.ae.process_packet(pkt1) - ret_val2 = self.ae.process_packet(pkt2) - logger.info('Checking index - [%s] - count - [%s]', - index, self.ae.packet_count) - if index * 2 < self.ae.packet_count: - logger.info('Expecting false - [%s]', ret_val1) - self.assertFalse(ret_val1) - self.assertFalse(ret_val2) - else: - logger.info('Expecting true - [%s]', ret_val1) - self.assertTrue(ret_val1) - self.assertTrue(ret_val2) - - def test_start_two_ipv6_tcp_attacks(self): - """ - Tests to ensure that two IPv6 UDP attacks have been triggered - :return: - """ - pkt1 = (Ether(src='00:00:00:00:01:01', dst=self.dst_mac, - type=consts.IPV6_TYPE) / - IPv6(dst=self.dst_ipv6, - src=self.src_ipv6, - nh=consts.UDP_PROTO) / - UdpInt() / - IntShim(length=9, next_proto=consts.TCP_PROTO) / - IntHeader(meta_len=1) / - IntMeta1(switch_id=3) / - IntMeta2(switch_id=2) / - SourceIntMeta(switch_id=1, orig_mac=self.orig_mac) / - TCP(dport=self.dport, sport=self.sport) / - 'hello transparent-security') - - pkt2 = (Ether(src='00:00:00:00:01:01', dst=self.dst_mac, - type=consts.IPV6_TYPE) / - IPv6(dst=self.dst_ipv6, - src=self.src_ipv6, - nh=consts.UDP_PROTO) / - UdpInt() / - IntShim(length=9, next_proto=consts.TCP_PROTO) / - IntHeader(meta_len=1) / - IntMeta1(switch_id=3) / - IntMeta2(switch_id=2) / - SourceIntMeta(switch_id=1, orig_mac=self.orig_mac) / - TCP(dport=self.dport, sport=self.sport) / - 'hello transparent-security') - - for index in range(0, self.ae.packet_count): - logger.info('Iteration #%s', index) - ret_val1 = self.ae.process_packet(pkt1) - ret_val2 = self.ae.process_packet(pkt2) - logger.info('Checking index - [%s] - count - [%s]', - index, self.ae.packet_count) - if index * 2 < self.ae.packet_count: - logger.info('Expecting false - [%s]', ret_val1) - self.assertFalse(ret_val1) - self.assertFalse(ret_val2) - else: - logger.info('Expecting true - [%s]', ret_val1) - self.assertTrue(ret_val1) - self.assertTrue(ret_val2) - - -def rand_mac(): - return "%02x:%02x:%02x:%02x:%02x:%02x" % ( - randint(0, 255), - randint(0, 255), - randint(0, 255), - randint(0, 255), - randint(0, 255), - randint(0, 255) - ) diff --git a/tests/trans_sec/controller/aggregate_controller_tests.py b/tests/trans_sec/controller/aggregate_controller_tests.py deleted file mode 100644 index f7323aa1..00000000 --- a/tests/trans_sec/controller/aggregate_controller_tests.py +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Unit tests for convert.py -import logging -import mock -import pkg_resources -import unittest - -import yaml - -from trans_sec.controller.aggregate_controller import AggregateController - -logger = logging.getLogger('aggregate_controller_tests') - - -class AggregateControllerTests(unittest.TestCase): - """ - Tests for the CoreController class - """ - def setUp(self): - # Parse topology file and store into object - topo_file = pkg_resources.resource_filename( - 'tests.trans_sec.conf', 'test_topology.yaml') - with open(topo_file, 'r') as f: - self.topo = yaml.load(f) - logger.info("Opened file - %s" % f.name) - - @mock.patch('trans_sec.p4runtime_lib.helper.P4InfoHelper', - return_value=mock.Mock()) - @mock.patch('trans_sec.p4runtime_lib.p4rt_switch.P4RuntimeSwitch', - return_value=mock.Mock()) - def test_construction(self, m1, m2): - """ - Tests constructor for class CoreController - """ - controller = AggregateController('bmv2', 'config_dir', self.topo, - '/tmp') - self.assertIsNotNone(controller) diff --git a/tests/trans_sec/controller/core_controller_tests.py b/tests/trans_sec/controller/core_controller_tests.py deleted file mode 100644 index 13913caa..00000000 --- a/tests/trans_sec/controller/core_controller_tests.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Unit tests for convert.py -import logging -import mock -import pkg_resources -import unittest - -import yaml - -from trans_sec.controller.core_controller import CoreController - -logger = logging.getLogger('core_controller_test') - - -class CoreControllerTests(unittest.TestCase): - """ - Tests for the CoreController class - """ - def setUp(self): - # Parse topology file and store into object - topo_file = pkg_resources.resource_filename( - 'tests.trans_sec.conf', 'test_topology.yaml') - with open(topo_file, 'r') as f: - self.topo = yaml.load(f) - logger.info("Opened file - %s" % f.name) - - @mock.patch('trans_sec.p4runtime_lib.helper.P4InfoHelper', - return_value=mock.Mock()) - @mock.patch('trans_sec.p4runtime_lib.p4rt_switch.P4RuntimeSwitch', - return_value=mock.Mock()) - def test_construction(self, m1, m2): - """ - Tests constructor for class CoreController - """ - controller = CoreController('bmv2', 'config_dir', self.topo, '/tmp') - self.assertIsNotNone(controller) diff --git a/tests/trans_sec/controller/ddos_sdn_controller_tests.py b/tests/trans_sec/controller/ddos_sdn_controller_tests.py deleted file mode 100644 index 395f6032..00000000 --- a/tests/trans_sec/controller/ddos_sdn_controller_tests.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Unit tests for http_session.py -import logging -import unittest - -import pkg_resources - -from trans_sec.controller.ddos_sdn_controller import DdosSdnController - -logger = logging.getLogger('ddos_sdn_controller_tests') - - -class HttpSessionTests(unittest.TestCase): - """ - Unit tests for utility functions in convert.py - """ - - # def setUp(self): - # """ - # Start HTTP server - # :return: - # """ - # logging.basicConfig(level=logging.DEBUG) - # topo_file = pkg_resources.resource_filename( - # 'tests.trans_sec.conf', 'test_topology.json') - # mock_switch_conf_dir = None - # self.controller = DdosSdnController( - # topo_file, mock_switch_conf_dir, 9998, 'scenario1', '/tmp') - # self.controller.start() - # - # def tearDown(self): - # self.controller.stop() - - # def test_foo(self): - # self.controller.add_attacker({}) diff --git a/tests/trans_sec/controller/gateway_controller_tests.py b/tests/trans_sec/controller/gateway_controller_tests.py deleted file mode 100644 index 40577df1..00000000 --- a/tests/trans_sec/controller/gateway_controller_tests.py +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Unit tests for convert.py -import logging -import mock -import pkg_resources -import unittest - -import yaml - -from trans_sec.controller.gateway_controller import GatewayController - -logger = logging.getLogger('gateway_controller_tests') - - -class GatewayControllerTests(unittest.TestCase): - """ - Tests for the CoreController class - """ - def setUp(self): - # Parse topology file and store into object - topo_file = pkg_resources.resource_filename( - 'tests.trans_sec.conf', 'test_topology.yaml') - with open(topo_file, 'r') as f: - self.topo = yaml.load(f) - logger.info("Opened file - %s" % f.name) - - @mock.patch('trans_sec.p4runtime_lib.helper.P4InfoHelper', - return_value=mock.Mock()) - @mock.patch('trans_sec.p4runtime_lib.p4rt_switch.P4RuntimeSwitch', - return_value=mock.Mock()) - def test_construction(self, m1, m2): - """ - Tests constructor for class CoreController - """ - controller = GatewayController('bmv2', 'config_dir', self.topo, '/tmp') - self.assertIsNotNone(controller) - controller.make_switch_rules(True) diff --git a/tests/trans_sec/mininet/__init__.py b/tests/trans_sec/mininet/__init__.py deleted file mode 100644 index e9b9345f..00000000 --- a/tests/trans_sec/mininet/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Unit tests for convert.py diff --git a/tests/trans_sec/p4runtime_lib/__init__.py b/tests/trans_sec/p4runtime_lib/__init__.py deleted file mode 100644 index 77c9835a..00000000 --- a/tests/trans_sec/p4runtime_lib/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/tests/trans_sec/p4runtime_lib/aggregate_switch_tests.py b/tests/trans_sec/p4runtime_lib/aggregate_switch_tests.py deleted file mode 100644 index 892711fb..00000000 --- a/tests/trans_sec/p4runtime_lib/aggregate_switch_tests.py +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import logging -import sys -import unittest -import mock -import pkg_resources -import yaml - -from trans_sec.p4runtime_lib.aggregate_switch import AggregateSwitch - -logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) -logger = logging.getLogger('aggregate_switch_tests') - - -class AggregateSwitchTests(unittest.TestCase): - """ - Tests for the CoreController class - """ - - def setUp(self): - topo_file = pkg_resources.resource_filename( - 'tests.trans_sec.conf', 'aggregate-topo.yaml') - with open(topo_file, 'r') as f: - topo_dict = yaml.safe_load(f) - - print(topo_dict) - self.sw_info = topo_dict['switches']['aggregate'] - self.p4_json = pkg_resources.resource_filename( - 'tests.trans_sec.conf', 'aggregate.json') - - @mock.patch('trans_sec.p4runtime_lib.helper.P4InfoHelper', - return_value=mock.Mock()) - @mock.patch( - 'trans_sec.switch.SwitchConnection.build_device_config', - return_value=mock.Mock()) - def test_construction(self, m1, m2): - """ - Tests constructor for class CoreController - """ - - switch = AggregateSwitch(sw_info=self.sw_info) - self.assertIsNotNone(switch) diff --git a/tests/trans_sec/p4runtime_lib/core_switch_tests.py b/tests/trans_sec/p4runtime_lib/core_switch_tests.py deleted file mode 100644 index dcb2588d..00000000 --- a/tests/trans_sec/p4runtime_lib/core_switch_tests.py +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import logging -import sys -import unittest -import mock -import pkg_resources -import yaml - -from trans_sec.p4runtime_lib.core_switch import CoreSwitch - -logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) -logger = logging.getLogger('core_switch_tests') - - -class CoreSwitchTests(unittest.TestCase): - """ - Tests for the CoreController class - """ - - def setUp(self): - topo_file = pkg_resources.resource_filename( - 'tests.trans_sec.conf', 'core-topo.yaml') - with open(topo_file, 'r') as f: - topo_dict = yaml.safe_load(f) - - print(topo_dict) - self.sw_info = topo_dict['switches']['core'] - self.p4_json = pkg_resources.resource_filename( - 'tests.trans_sec.conf', 'core.json') - - @mock.patch( - 'trans_sec.p4runtime_lib.helper.P4InfoHelper', - return_value=mock.Mock()) - @mock.patch( - 'trans_sec.switch.SwitchConnection.build_device_config', - return_value=mock.Mock()) - def test_construction(self, m1, m2): - """ - Tests constructor for class CoreController - """ - - switch = CoreSwitch(sw_info=self.sw_info) - self.assertIsNotNone(switch) diff --git a/tests/trans_sec/p4runtime_lib/gateway_switch_tests.py b/tests/trans_sec/p4runtime_lib/gateway_switch_tests.py deleted file mode 100644 index b3be31a3..00000000 --- a/tests/trans_sec/p4runtime_lib/gateway_switch_tests.py +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import logging -import sys -import unittest -import mock -import pkg_resources -import yaml - -from trans_sec.p4runtime_lib.gateway_switch import GatewaySwitch - -logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) -logger = logging.getLogger('gateway_switch_tests') - - -class GatewaySwitchTests(unittest.TestCase): - """ - Tests for the CoreController class - """ - - def setUp(self): - topo_file = pkg_resources.resource_filename( - 'tests.trans_sec.conf', 'gateway-topo.yaml') - with open(topo_file, 'r') as f: - topo_dict = yaml.safe_load(f) - - print(topo_dict) - self.sw_info = topo_dict['switches']['gateway'] - self.p4_json = pkg_resources.resource_filename( - 'tests.trans_sec.conf', 'gateway.json') - - @mock.patch('trans_sec.p4runtime_lib.helper.P4InfoHelper', - return_value=mock.Mock()) - @mock.patch( - 'trans_sec.switch.SwitchConnection.build_device_config', - return_value=mock.Mock()) - def test_construction(self, m1, m2): - """ - Tests constructor for class CoreController - """ - - switch = GatewaySwitch(sw_info=self.sw_info) - self.assertIsNotNone(switch) diff --git a/trans_sec/analytics/__init__.py b/trans_sec/analytics/__init__.py deleted file mode 100644 index e9b9345f..00000000 --- a/trans_sec/analytics/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Unit tests for convert.py diff --git a/trans_sec/analytics/oinc.py b/trans_sec/analytics/oinc.py deleted file mode 100644 index eb2767ec..00000000 --- a/trans_sec/analytics/oinc.py +++ /dev/null @@ -1,626 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import abc -import datetime -import ipaddress -import logging -import threading -import time - -from time import strftime, localtime -from anytree import search, Node, RenderTree -from scapy.all import sniff -from scapy.layers.inet import IP, UDP, TCP -from scapy.layers.inet6 import IPv6 -from scapy.layers.l2 import Ether - -from trans_sec.consts import (UDP_PROTO, UDP_TRPT_DST_PORT, IPV4_TYPE, - UDP_INT_DST_PORT, IPV6_TYPE) -from trans_sec.packet.inspect_layer import ( - IntHeader, IntMeta1, IntMeta2, IntShim, SourceIntMeta, TelemetryReport, - EthInt, DropReport) -from trans_sec.utils import tps_utils - -logger = logging.getLogger('oinc') - - -class PacketAnalytics(object): - """ - Analytics Engine class - """ - - def __init__(self, sdn_interface, packet_count=100, sample_interval=60, - sdn_attack_context='gwAttack'): - """ - Constructor - :param sdn_interface: the HTTP interface to the SDN Controller - :param packet_count: the number of packets to trigger an attack - :param sample_interval: the interval in seconds used for counting the - packets - """ - self.sdn_interface = sdn_interface - self.packet_count = packet_count - self.sample_interval = sample_interval - self.count_map = dict() - self.bytes_map = dict() - self.sniff_stop = threading.Event() - self.sdn_attack_context = sdn_attack_context - - logger.debug("Started AE with attack call to [%s/%s]", - self.sdn_interface, self.sdn_attack_context) - - def start_sniffing(self, iface, drop_iface=None, - udp_dport=UDP_TRPT_DST_PORT): - """ - Starts the sniffer thread - :param iface: the interface to sniff for telemetry reports - :param drop_iface: the interface to sniff for drop reports - :param udp_dport: the UDP dport to sniff (default 555) - """ - logger.info('Starting INT packet listener int_thread') - int_thread = threading.Thread(target=self.sniff_int, - args=(iface, udp_dport)) - int_thread.start() - logger.debug('int_thread has been started') - - if drop_iface: - logger.info('Starting Drop Report listener dr_thread') - dr_thread = threading.Thread(target=self.sniff_drop, - args=(drop_iface, 'foo')) - dr_thread.start() - logger.debug('dr_thread has been started') - - def sniff_int(self, iface, udp_port): - logger.info("INT monitoring iface [%s]", iface) - - try: - sniff(iface=iface, - prn=lambda packet: self.handle_packet(packet, udp_port), - stop_filter=lambda p: self.sniff_stop.is_set()) - except Exception as e: - logger.error('Unexpected error sniffing for INT Data - [%s]', e) - self.sniff_int(iface, udp_port) - - def sniff_drop(self, iface, foo=None): - logger.info("Drop monitoring iface [%s]", iface) - - try: - sniff(iface=iface, - prn=lambda packet: self.handle_drop_rpt(packet), - stop_filter=lambda p: self.sniff_stop.is_set()) - except Exception as e: - logger.error('Unexpected error sniffing for Drop Reports - [%s]', - e) - raise e - - def stop_sniffing(self): - """ - Stops the sniffer thread - """ - self.sniff_stop.set() - - def handle_packet(self, packet, udp_dport): - """ - Determines whether or not to process this packet - :param packet: the packet to process - :param udp_dport: the UDP protocol dport value to filter - :return T/F - True when an attack has been triggered - """ - return self.process_packet(packet, udp_dport) - - def handle_drop_rpt(self, packet): - """ - Determines whether or not to process this packet - :param packet: the packet to process - :return T/F - True when an attack has been triggered - """ - return self.process_drop_rpt(packet) - - def _send_attack(self, **attack_dict): - """ - Sends an HTTP POST to the SDN controllers HTTP interface 'attack' - :param attack_dict: the data to send - :raises Exception: due to the remote HTTP POST - """ - logger.info('Start attack - [%s] to [%s]', - attack_dict, self.sdn_attack_context) - self.sdn_interface.post(self.sdn_attack_context, attack_dict) - - def _stop_attack(self, **attack_dict): - """ - Sends an HTTP POST to the SDN controllers HTTP interface 'attack' - :param attack_dict: the data to send - :raises Exception: due to the remote HTTP POST - """ - logger.info('Stopping attack - [%s] to [%s]', - attack_dict, self.sdn_attack_context) - self.sdn_interface.delete(self.sdn_attack_context, attack_dict) - logger.info('Attack stopped at time [%s] with args - [%s]', - strftime("%b-%d-%Y %H:%M:%S", localtime()), attack_dict) - - @abc.abstractmethod - def process_packet(self, packet, udp_dport=UDP_INT_DST_PORT): - """ - Processes a packet to determine if an attack is occurring - :param packet: the packet to process - :param udp_dport: the UDP port value on which to filter - :return: T/F - True when an attack has been triggered - """ - return - - @abc.abstractmethod - def process_drop_rpt(self, packet): - """ - Processes a drop packet to determine if an attack no longer occurring - :param packet: the packet to process - :return: T/F - True when an attack should no longer be triggered - """ - return - - -def extract_int_data(ether_pkt): - """ - Parses the required data from the packet - :param ether_pkt: the packet to parse - :return: dict with choice header fields extracted - """ - logger.debug('Extracting packet - [%s]', ether_pkt.summary()) - if ether_pkt.type == IPV4_TYPE: - ip_pkt = IP(_pkt=ether_pkt.payload) - logger.debug('IPv4 dst - [%s], src - [%s], proto - [%s]', - ip_pkt.dst, ip_pkt.src, ip_pkt.proto) - elif ether_pkt.type == IPV6_TYPE: - ip_pkt = IPv6(_pkt=ether_pkt.payload) - logger.debug('IPv6 dst - [%s], src - [%s], nh - [%s]', - ip_pkt.dst, ip_pkt.src, ip_pkt.nh) - else: - logger.warning('Unable to process ether type - [%s]', ether_pkt.type) - return None - - udp_int_pkt = UDP(_pkt=ip_pkt.payload) - logger.debug('UDP INT sport - [%s], dport - [%s], len - [%s]', - udp_int_pkt.sport, udp_int_pkt.dport, udp_int_pkt.len) - int_shim_pkt = IntShim(_pkt=udp_int_pkt.payload) - logger.debug('INT Shim next_proto - [%s], npt - [%s], length - [%s]', - int_shim_pkt.next_proto, int_shim_pkt.npt, - int_shim_pkt.length) - int_hdr_pkt = IntHeader(_pkt=int_shim_pkt.payload) - - logger.debug('INT Header ver - [%s]', int_hdr_pkt.ver) - - if int_shim_pkt.length == 7: - source_int_pkt = SourceIntMeta(_pkt=int_hdr_pkt.payload) - elif int_shim_pkt.length == 8: - int_meta_1 = IntMeta1(_pkt=int_hdr_pkt.payload) - logger.debug('INT Meta 1 switch_id - [%s]', int_meta_1.switch_id) - source_int_pkt = SourceIntMeta(_pkt=int_meta_1.payload) - elif int_shim_pkt.length == 9: - int_meta_1 = IntMeta1(_pkt=int_hdr_pkt.payload) - logger.debug('INT Meta 1 switch_id - [%s]', int_meta_1.switch_id) - int_meta_2 = IntMeta2(_pkt=int_meta_1.payload) - logger.debug('INT Meta 2 switch_id - [%s]', int_meta_2.switch_id) - source_int_pkt = SourceIntMeta(_pkt=int_meta_2.payload) - else: - return - - logger.debug('SourceIntMeta switch_id - [%s], orig_mac - [%s]', - source_int_pkt.switch_id, source_int_pkt.orig_mac) - - if int_shim_pkt.next_proto == UDP_PROTO: - tcp_udp_pkt = UDP(_pkt=source_int_pkt.payload) - logger.debug('TCP sport - [%s], dport - [%s], len - [%s]', - tcp_udp_pkt.sport, tcp_udp_pkt.dport, tcp_udp_pkt.len) - else: - tcp_udp_pkt = TCP(_pkt=source_int_pkt.payload) - logger.debug('TCP sport - [%s], dport - [%s]', - tcp_udp_pkt.sport, tcp_udp_pkt.dport) - - orig_mac = source_int_pkt.orig_mac - - try: - out = dict( - devMac=orig_mac, - devAddr=ip_pkt.src, - dstAddr=ip_pkt.dst, - dstPort=tcp_udp_pkt.dport, - protocol=int_shim_pkt.next_proto, - packetLen=len(ether_pkt), - ) - except Exception as e: - logger.error('Error extracting header data - %s', e) - return None - logger.debug('Extracted header data [%s]', out) - return out - - -def extract_trpt_data(udp_packet): - """ - Parses the required data from the packet - :param udp_packet: the packet to parse - :return: dict with choice header fields extracted - """ - trpt_pkt = TelemetryReport(_pkt=udp_packet.payload) - trpt_eth = EthInt(trpt_pkt.payload) - logger.debug('TRPT ethernet dst - [%s], src - [%s], type - [%s]', - trpt_eth.dst, trpt_eth.src, trpt_eth.type) - return extract_int_data(trpt_eth) - - -def extract_drop_rpt(udp_packet): - """ - Parses the required data from the packet - :param udp_packet: the packet to parse - :return: tuple of key (hash|int) and value (count|int) - """ - logger.debug('UDP packet sport [%s], dport [%s], len [%s]', - udp_packet.sport, udp_packet.dport, udp_packet.len) - - drop_rpt = DropReport(_pkt=udp_packet.payload) - if drop_rpt: - return drop_rpt.drop_hash, drop_rpt.drop_count - - -class Oinc(PacketAnalytics): - """ - Oinc implementation of PacketAnalytics - """ - - def __init__(self, sdn_interface, packet_count=100, sample_interval=60): - super(self.__class__, self).__init__(sdn_interface, packet_count, - sample_interval) - self.tree = Node('root') - - def process_packet(self, packet, udp_dport=UDP_INT_DST_PORT): - mac, src_ip, dst_ip, dst_port, packet_size = self.__parse_tree(packet) - - if mac: - if src_ip and dst_ip and dst_port and packet_size: - self.__packet_with_mac(mac, src_ip, dst_ip, dst_port, - packet_size) - self.__manage_tree() - - def process_drop_rpt(self, packet): - pass - - def __parse_tree(self, packet): - """ - Processes a packet from a new device that has not been counted - """ - info = extract_int_data(packet[Ether]) - logger.info('Processing packet with info [%s]', info) - - macs = search.findall_by_attr(self.tree, info.get('srcMac'), - name='name', maxlevel=2, maxcount=1) - - mac = None - src_ip = None - dst_ip = None - dst_port = None - packet_size = None - - if len(macs) > 0: - mac = macs[0] - src_ips = search.findall_by_attr( - mac, info.get('srcIP'), name='name', maxlevel=2, maxcount=1) - if len(src_ips) != 0: - src_ip = src_ips[0] - dst_ips = search.findall_by_attr( - src_ip, info.get('dstIP'), name='name', maxlevel=2, - maxcount=1) - if len(dst_ips) != 0: - dst_ip = dst_ips[0] - logger.info('Processing source IPs - %s', src_ips) - dst_ports = search.findall_by_attr( - dst_ip, info.get('dstPort'), name='name', - maxlevel=2, maxcount=1) - if len(dst_ports) != 0: - dst_port = dst_ports[0] - packet_sizes = search.findall_by_attr( - dst_port, info.get('packet_size'), - name='name', maxlevel=2, maxcount=1) - if len(packet_sizes) != 0: - packet_size = packet_sizes[0] - - return mac, src_ip, dst_ip, dst_port, packet_size - - def __manage_tree(self): - """ - Updates the tree - I don't think this routine does anything at all - """ - for pre, fill, node in RenderTree(self.tree): - if node.name == 'count': - logger.info( - "Tree info %s%s: %s %s p/s attack: %s", - pre, node.name, node.value, node.pps, node.attack) - else: - logger.info("Pre - [%s], Fill - [%s], Node - [%s]", - pre, fill, node.name) - - def __packet_with_mac(self, mac, src_ip, dst_ip, dst_port, packet_size): - """ - Processes a packet from an existing device that has been counted - """ - logger.debug('Packet with MAC [%s] and source IP [%s]', mac, src_ip) - count = packet_size.children[0] - count.value = count.value + 1 - base_time = count.time - current_time = datetime.datetime.today() - delta = (current_time - base_time).total_seconds() - count.pps = count.value / delta - if (count.value > 3 and count.pps > 100 - and not count.attack): - logger.info('UDP Flood attack detected') - count.attack = True - - # Send to SDN - try: - self._send_attack(**dict( - src_mac=mac.name, - dst_ip=dst_ip.name, - dst_port=dst_port.name)) - except Exception as e: - logger.error('Unexpected error [%s]', e) - - if delta > 60: - count.time = current_time - count.value = 1 - - -class SimpleAE(PacketAnalytics): - """ - Simple implementation of PacketAnalytics where the count for detecting - attack notifications is based on the unique hash of the extracted INT data - """ - - def __init__(self, sdn_interface, packet_count=100, sample_interval=60, - sdn_attack_context='gwAttack', drop_count=3, - byte_count=50000): - super(self.__class__, self).__init__( - sdn_interface, packet_count, sample_interval, sdn_attack_context) - # Holds the last time an attack call was issued to the SDN controller - self.attack_map = dict() - self.attack_payload = dict() - - # same key as above, value tuple (int, int) first holds the drop count - # and the second holds the number of drop reports without receiving - # any associated packets - self.drop_rpt_map = dict() - self.drop_count = drop_count - self.byte_count = byte_count - - def process_packet(self, packet, udp_dport=UDP_INT_DST_PORT): - """ - Processes a packet to determine if an attack is occurring if the IP - protocol is as expected - :param packet: the packet to process - :param udp_dport: the UDP port value on which to filter - :return: T/F - True when an attack has been triggered - """ - ip_pkt, protocol, pkt_bytes = parse_ip_pkt(packet) - if ip_pkt and protocol and protocol == UDP_PROTO: - udp_packet = UDP(_pkt=ip_pkt.payload) - if udp_packet.dport == udp_dport and udp_dport == UDP_INT_DST_PORT: - int_data = extract_int_data(packet[Ether]) - if int_data: - return self.__process(int_data, pkt_bytes) - else: - logger.warning('Unable to debug INT data') - return False - elif (udp_packet.dport == udp_dport - and udp_dport == UDP_TRPT_DST_PORT): - int_data = extract_trpt_data(udp_packet) - if int_data: - return self.__process(int_data, pkt_bytes) - else: - logger.warning('Unable to debug INT data') - return False - else: - logger.debug( - 'Cannot process UDP packet dport of - [%s], expected - ' - '[%s]', udp_packet.dport, udp_dport) - return False - - def __process(self, int_data, pkt_bytes): - """ - Processes INT data for analysis - :param int_data: the data to process - :return: - """ - ip_addr = ipaddress.ip_address(int_data['dstAddr']) - ipv4 = '0.0.0.0' - ipv6 = '::' - if ip_addr.version == 4: - ipv4 = str(ip_addr) - else: - ipv6 = str(ip_addr) - attack_map_key = tps_utils.create_attack_hash( - mac=int_data['devMac'], port=int_data['dstPort'], ip_addr=ipv4, - ipv6_addr=ipv6) - logger.debug('Attack map key - [%s]', attack_map_key) - - if not self.drop_rpt_map.get(attack_map_key): - logger.debug( - 'Updating drop_rpt_map entry with zeros at time [%s] - [%s]', - strftime("%b-%d-%Y %H:%M:%S", localtime()), attack_map_key) - self.drop_rpt_map[attack_map_key] = (0, 0) - else: - logger.debug('Creating drop_rpt_map entry with key - [%s]', - attack_map_key) - tuple_val = self.drop_rpt_map[attack_map_key] - new_tuple = (tuple_val[0], 0) - self.drop_rpt_map[attack_map_key] = new_tuple - - if not self.count_map.get(attack_map_key): - self.count_map[attack_map_key] = list() - - curr_time = datetime.datetime.now() - self.count_map.get(attack_map_key).append((curr_time, pkt_bytes)) - time_bytes_tuple = self.count_map.get(attack_map_key) - count = 0 - total_bytes = 0 - for eval_time, pkt_bytes in time_bytes_tuple: - logger.debug('eval_time - [%s], pkt_bytes - [%s]', - eval_time, pkt_bytes) - delta = (curr_time - eval_time).total_seconds() - if delta > self.sample_interval: - time_bytes_tuple.remove((eval_time, pkt_bytes)) - else: - count += 1 - total_bytes += pkt_bytes - logger.info( - 'Total bytes [%s] and packet count [%s] received in window', - total_bytes, count) - if count > self.packet_count or total_bytes > self.byte_count: - logger.warning( - 'Attack detected at time [%s]- count [%s] & bytes [%s] ' - 'with key [%s]', - strftime("%b-%d-%Y %H:%M:%S", localtime()), - count, total_bytes, attack_map_key) - - attack_dict = dict( - src_mac=int_data['devMac'], - dst_port=int_data['dstPort'], - dst_ip=int_data['dstAddr']) - - # Send to SDN - last_attack = self.attack_map.get(attack_map_key) - if not last_attack or time.time() - last_attack > 1: - logger.info('Calling SDN, last attack sent - [%s]', - last_attack) - try: - self.attack_map[attack_map_key] = time.time() - self.attack_payload[attack_map_key] = attack_dict - self._send_attack(**attack_dict) - return True - except Exception as e: - logger.error('Unexpected error [%s]', e) - return False - else: - logger.debug( - 'Not calling SDN as last attack notification for %s' - ' was only %s seconds ago', - attack_dict, time.time() - last_attack) - return True - else: - logger.debug('No attack detected - count [%s]', count) - return False - - def process_drop_rpt(self, packet): - """ - Processes a drop report packet to determine if an attack stopped - :param packet: the packet to process - :return: T/F - True when an attack has been triggered - """ - ip_pkt, protocol, pkt_bytes = parse_ip_pkt(packet) - if ip_pkt and protocol and protocol == UDP_PROTO: - udp_packet = UDP(_pkt=ip_pkt.payload) - if udp_packet.dport == UDP_TRPT_DST_PORT: - hash_key, drop_count = extract_drop_rpt(udp_packet) - if self.drop_rpt_map.get(hash_key): - tuple_val = self.drop_rpt_map[hash_key] - old_count = tuple_val[0] - if old_count != drop_count: - new_tuple = (drop_count, 0) - self.drop_rpt_map[hash_key] = new_tuple - return False - else: - new_tuple = (tuple_val[0], tuple_val[1] + 1) - self.drop_rpt_map[hash_key] = new_tuple - if new_tuple[1] >= self.drop_count: - attack_body = self.attack_payload.get(hash_key) - if attack_body: - logger.info('Stopping attack with [%s]', - attack_body) - self._stop_attack(**attack_body) - - logger.info('Clearing maps after attacked stopped') - self.attack_payload[hash_key] = None - self.attack_map[hash_key] = None - self.count_map[hash_key] = None - self.drop_rpt_map[hash_key] = None - logger.info( - 'Cleared maps with hash [%s] at time [%s]', - hash_key, strftime("%b-%d-%Y %H:%M:%S", - localtime())) - return True - else: - self.drop_rpt_map[hash_key] = new_tuple - return False - return False - - -def parse_ip_pkt(packet): - ip_pkt = None - protocol = None - pkt_bytes = 0 - try: - if packet[Ether].type == IPV4_TYPE: - ip_pkt = IP(_pkt=packet[Ether].payload) - protocol = ip_pkt.proto - pkt_bytes = ip_pkt.len - elif packet[Ether].type == IPV6_TYPE: - ip_pkt = IPv6(_pkt=packet[Ether].payload) - protocol = ip_pkt.nh - pkt_bytes = ip_pkt.plen - except Exception as e: - logger.error('Unexpected error processing packet - [%s]', e) - - return ip_pkt, protocol, pkt_bytes - - -class IntLoggerAE(PacketAnalytics): - """ - Logs only INT packets - """ - - def process_packet(self, packet, udp_dport=UDP_INT_DST_PORT): - """ - Logs the INT data within the packet - :param packet: the INT packet - :param udp_dport: the UDP port value on which to filter - :return: False - """ - logger.info('INT Packet data - [%s]', extract_int_data(packet[Ether])) - return False - - def process_drop_rpt(self, packet): - pass - - -class LoggerAE(PacketAnalytics): - """ - Logging only - """ - - def handle_packet(self, packet, ip_proto=None): - """ - Logs every received packet's summary data - :param packet: extracts data from here - :param ip_proto: does nothing here - :return: False - """ - logger.info('Packet data - [%s]', packet.summary()) - return False - - def process_packet(self, packet, udp_dport=UDP_INT_DST_PORT): - """ - No need to implement - :param packet: the packet that'll never come in - :param udp_dport: the UDP port value on which to filter - :raises NotImplemented - """ - raise NotImplemented - - def process_drop_rpt(self, packet): - pass diff --git a/trans_sec/bfruntime_lib/gateway_switch.py b/trans_sec/bfruntime_lib/gateway_switch.py deleted file mode 100644 index 820fcb69..00000000 --- a/trans_sec/bfruntime_lib/gateway_switch.py +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright (c) 2020 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Originally copied from: -# -# Copyright 2017-present Open Networking Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import logging -from abc import ABC - -from trans_sec.bfruntime_lib.bfrt_switch import BFRuntimeSwitch - -logger = logging.getLogger('gateway_switch') - - -class GatewaySwitch(BFRuntimeSwitch, ABC): - def __init__(self, sw_info, client_id=0, is_master=True): - """ - Construct Switch class to control BMV2 switches running gateway.p4 - """ - super(self.__class__, self).__init__(sw_info, client_id, is_master) - self.nat_udp_ports = set() - self.nat_tcp_ports = set() - self.tcp_port_count = 1 - self.udp_port_count = 1 - - def add_data_inspection(self, dev_id, dev_mac): - raise NotImplementedError - - def write_multicast_entry(self, hosts): - raise NotImplementedError diff --git a/trans_sec/controller/aggregate_controller.py b/trans_sec/controller/aggregate_controller.py index 365e5570..c3549be2 100755 --- a/trans_sec/controller/aggregate_controller.py +++ b/trans_sec/controller/aggregate_controller.py @@ -11,28 +11,14 @@ # See the License for the specific language governing permissions and # limitations under the License. from logging import getLogger +from trans_sec.bfruntime_lib.aggregate_switch import ( + AggregateSwitch as BFRTSwitch) from trans_sec.controller.abstract_controller import AbstractController from trans_sec.controller.ddos_sdn_controller import AGG_CTRL_KEY from trans_sec.utils import tps_utils logger = getLogger('aggregate_controller') -try: - from trans_sec.p4runtime_lib.aggregate_switch import ( - AggregateSwitch as P4RTSwitch) -except Exception as e: - logger.warning( - "Error [%s] - while attempting to import " - "trans_sec.p4runtime_lib.aggregate_switch.AggregateSwitch", e) - -try: - from trans_sec.bfruntime_lib.aggregate_switch import ( - AggregateSwitch as BFRTSwitch) -except Exception as e: - logger.warning( - "Error [%s] - while attempting to import " - "trans_sec.bfruntime_lib.aggregate_switch.AggregateSwitch", e) - class AggregateController(AbstractController): """ @@ -48,14 +34,8 @@ def __init__(self, platform, p4_build_out, topo, log_dir, load_p4=True): self.attack_dict = {} def instantiate_switch(self, sw_info): - if 'arch' in sw_info and sw_info['arch'] == 'tna': - logger.info('Instantiating BFRT AggregateSwitch') - return BFRTSwitch(sw_info=sw_info) - else: - return P4RTSwitch( - sw_info=sw_info, - proto_dump_file='{}/{}-switch-controller.log'.format( - self.log_dir, sw_info['name'])) + logger.info('Instantiating BFRT AggregateSwitch') + return BFRTSwitch(sw_info=sw_info) def __get_agg_switch(self): return self.switches[0] diff --git a/trans_sec/controller/appcontroller.py b/trans_sec/controller/appcontroller.py deleted file mode 100644 index e17350b2..00000000 --- a/trans_sec/controller/appcontroller.py +++ /dev/null @@ -1,127 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import logging -import subprocess - -from trans_sec.controller.shortest_path import ShortestPath - -logger = logging.getLogger('appcontroller') - - -class AppController: - - def __init__(self, manifest=None, target=None, topo=None, net=None, - links=None): - self.manifest = manifest - self.target = target - self.conf = manifest['targets'][target] - self.topo = topo - self.net = net - self.links = links - - @staticmethod - def read_entries(filename): - entries = [] - with open(filename, 'r') as f: - for line in f: - line = line.strip() - if line == '': - continue - entries.append(line) - return entries - - @staticmethod - def add_entries(thrift_port=9090, sw=None, entries=None): - assert entries - if sw: - thrift_port = sw.thrift_port - - p = subprocess.Popen( - ['simple_switch_CLI', '--thrift-port', str(thrift_port)], - stdin=subprocess.PIPE) - p.communicate(input='\n'.join(entries)) - - @staticmethod - def read_register(register, idx, thrift_port=9090, sw=None): - if sw: - thrift_port = sw.thrift_port - p = subprocess.Popen( - ['simple_switch_CLI', '--thrift-port', str(thrift_port)], - stdin=subprocess.PIPE, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - stdout, stderr = p.communicate( - input="register_read %s %d" % (register, idx)) - reg_val = list(filter(lambda l: ' %s[%d]' % (register, idx) in l, - stdout.split('\n')))[0].split('= ', 1)[1] - return int(reg_val) - - def start(self): - shortest_path = ShortestPath(self.links) - entries = {} - for sw in self.topo.switches(): - entries[sw] = [] - if ('switches' in self.conf - and sw in self.conf['switches'] - and 'entries' in self.conf['switches'][sw]): - extra_entries = self.conf['switches'][sw]['entries'] - if type(extra_entries) == list: # array of entries - entries[sw] += extra_entries - else: # path to file that contains entries - entries[sw] += self.read_entries(extra_entries) - - for host_name in self.topo.host_links: - host = self.net.get(host_name) - for link in self.topo.host_links[host_name].values(): - iface = host.intfNames()[link['idx']] - # use mininet to set ip and mac to let it know the change - host.setIP(link['host_ip'], 24) - host.setMAC(link['host_mac']) - host.cmd('arp -i %s -s %s %s' % ( - iface, link['sw_ip'], link['sw_mac'])) - host.cmd('ethtool --offload %s rx off tx off' % iface) - host.cmd('ip route add %s dev %s' % (link['sw_ip'], iface)) - - # TODO - determine why this was outside of the for block? - host.setDefaultRoute("via %s" % link['sw_ip']) - - for host in self.net.hosts: - for sw in self.net.switches: - path = shortest_path.get(sw.name, host.name, - exclude=lambda n: n[0] == 'h') - if not path: - continue - if not path[1][0] == 's': - continue # next hop is a switch - - for h2 in self.net.hosts: - if host == h2: - continue - path = shortest_path.get(host.name, h2.name, - exclude=lambda n: n[0] == 'h') - if not path: - continue - h_link = self.topo.host_links[host.name][path[1]] - h2_link = self.topo.host_links[h2.name].values()[0] - host.cmd('ip route add %s via %s' % ( - h2_link['host_ip'], h_link['sw_ip'])) - - logger.info("Configuring entries in p4 tables") - for sw_name in entries: - logger.info("Configuring switch... %s", sw_name) - sw = self.net.get(sw_name) - if entries[sw_name]: - self.add_entries(sw=sw, entries=entries[sw_name]) - logger.info("Configuration complete.") - - def stop(self): - pass diff --git a/trans_sec/controller/core_controller.py b/trans_sec/controller/core_controller.py index 889792b1..fcba4747 100755 --- a/trans_sec/controller/core_controller.py +++ b/trans_sec/controller/core_controller.py @@ -15,24 +15,10 @@ from trans_sec.controller.abstract_controller import AbstractController from trans_sec.controller.ddos_sdn_controller import CORE_CTRL_KEY -logger = getLogger('core_controller') +from trans_sec.bfruntime_lib.core_switch import (CoreSwitch as BFRTSwitch) -try: - logger.debug('Importing CoreSwitch as P4RTSwitch') - from trans_sec.p4runtime_lib.core_switch import CoreSwitch as P4RTSwitch - logger.debug('Imported CoreSwitch as P4RTSwitch') -except Exception as e: - logger.warning( - "Error [%s] - while attempting to import " - "trans_sec.p4runtime_lib.core_switch.CoreSwitch", e) -try: - logger.debug('Importing CoreSwitch as BFRTSwitch') - from trans_sec.bfruntime_lib.core_switch import ( - CoreSwitch as BFRTSwitch) - logger.debug('Imported CoreSwitch as BFRTSwitch') -except Exception as e: - logger.warning('Could not import bfrt classes') +logger = getLogger('core_controller') class CoreController(AbstractController): @@ -46,14 +32,8 @@ def __init__(self, platform, p4_build_out, topo, log_dir, load_p4=True): def instantiate_switch(self, sw_info): logger.info('Instantiating switch with arch - [%s]', sw_info) - if 'arch' in sw_info and sw_info['arch'] == 'tna': - logger.info('Instantiating BFRT CoreSwitch') - return BFRTSwitch(sw_info=sw_info) - else: - return P4RTSwitch( - sw_info=sw_info, - proto_dump_file='{}/{}-switch-controller.log'.format( - self.log_dir, sw_info['name'])) + logger.info('Instantiating BFRT CoreSwitch') + return BFRTSwitch(sw_info=sw_info) def __get_core_switch(self): return self.switches[0] diff --git a/trans_sec/controller/ddos_sdn_controller.py b/trans_sec/controller/ddos_sdn_controller.py index 4b3e8c1e..7daaa376 100755 --- a/trans_sec/controller/ddos_sdn_controller.py +++ b/trans_sec/controller/ddos_sdn_controller.py @@ -36,7 +36,6 @@ AGG_CTRL_KEY = 'aggregate' CORE_CTRL_KEY = 'core' -GATEWAY_CTRL_KEY = 'gateway' class DdosSdnController: @@ -228,35 +227,6 @@ def __data_inspection(self, di_req, del_flag=False): logger.warning('Could not find switch with device_id - [%s]', di_req['device_id']) - def remove_attacker(self, attack): - """ - Removes a device to mitigate an attack - :param attack: dict of attack - """ - host, gw_controller = self.__get_attack_host(attack) - logger.info('Adding attack to gateways with host - [%s]', host) - try: - gw_controller.remove_attacker(attack, host) - except Exception as e: - logger.error( - 'Error removing attacker to host - [%s] with error - [%s])', - host, e) - - def add_attacker(self, attack): - """ - Adds a device to mitigate an attack - :param attack: dict of attack - """ - host, gw_controller = self.__get_attack_host(attack) - logger.info('Adding attack to gateways with host - [%s]', host) - try: - gw_controller.add_attacker(attack, host) - except Exception as e: - logger.error( - 'Error adding gateway attacker to host - [%s] ' - 'with error - [%s])', host, e) - raise e - def remove_agg_attacker(self, attack): """ Removes a device to mitigate an attack @@ -386,38 +356,6 @@ def get_core_controller(self): core_controller = self.controllers.get(CORE_CTRL_KEY) return core_controller - def __get_attack_host(self, attack): - """ - Returns the host value or None - :param attack: - :return: - """ - gateway_controller = self.controllers.get(GATEWAY_CTRL_KEY) - if gateway_controller: - logger.info('Attack received - %s', attack) - - conditions = {'mac': attack['src_mac']} - logger.debug('Created conditions - [%s]', conditions) - values = self.topo.get('hosts').values() - logger.debug('Creating host with values - [%s]', values) - host = list(filter( - lambda item: all( - (item[k] == v for (k, v) in conditions.items())), - values)) - - logger.debug( - 'Check the hosts and register the attack with host object ' - '- [%s]', host) - logger.debug('host.__class__ - [%s]', host.__class__) - if len(host) > 0: - logger.debug('host len is - [%s]', len(host)) - return host[0], gateway_controller - else: - logger.error('No Device Matches MAC [%s]', - attack.get('src_mac')) - else: - logger.warning('No Gateway Controller call') - def __main_loop(self): """ Starts polling thread/Error adding attacker to host diff --git a/trans_sec/controller/gateway_controller.py b/trans_sec/controller/gateway_controller.py deleted file mode 100755 index 38a29fcd..00000000 --- a/trans_sec/controller/gateway_controller.py +++ /dev/null @@ -1,143 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import ipaddress -from logging import getLogger -from trans_sec.controller.abstract_controller import AbstractController -from trans_sec.controller.ddos_sdn_controller import GATEWAY_CTRL_KEY - -logger = getLogger('gateway_controller') - -try: - from trans_sec.p4runtime_lib.gateway_switch import ( - GatewaySwitch as P4RTSwitch) -except Exception as e: - logger.warning( - "Error [%s] - while attempting to import " - "trans_sec.p4runtime_lib.aggregate_switch.AggregateSwitch", e) - -try: - from trans_sec.bfruntime_lib.gateway_switch import ( - GatewaySwitch as BFRTSwitch) -except Exception as e: - logger.warning( - "Error [%s] - while attempting to import " - "trans_sec.bfruntime_lib.aggregate_switch.AggregateSwitch", e) - -try: - from trans_sec.p4runtime_lib.gateway_switch import ( - GatewaySwitch as P4RTSwitch) -except Exception as e: - logger.warning( - "Error [%s] - while attempting to import " - "trans_sec.p4runtime_lib.gateway_switch.GatewaySwitch", e) - -try: - from trans_sec.bfruntime_lib.gateway_switch import ( - GatewaySwitch as BFRTSwitch) -except Exception as e: - logger.warning('Could not import bfrt classes') - - -class GatewayController(AbstractController): - """ - Implementation of the controller for a switch running the gateway.p4 - program - """ - def __init__(self, platform, p4_build_out, topo, log_dir, load_p4=True): - super(self.__class__, self).__init__( - platform, p4_build_out, topo, GATEWAY_CTRL_KEY, log_dir, load_p4) - - def instantiate_switch(self, sw_info): - if 'arch' in sw_info and sw_info['arch'] == 'tna': - return BFRTSwitch(sw_info=sw_info) - else: - return P4RTSwitch( - sw_info=sw_info, - proto_dump_file='{}/{}-switch-controller.log'.format( - self.log_dir, sw_info['name'])) - - def make_rules(self, sw, north_facing_links, south_facing_links, - add_di): - """ - Overrides the abstract method from super - :param sw: switch object - :param north_facing_links: northbound links - :param south_facing_links: southbound links - :param add_di: when True inserts into the data_inspection_t table - """ - for device_link in south_facing_links: - device = self.topo['hosts'].get(device_link['south_node']) - - if device: - logger.info('Gateway: ' + sw.name + - ' connects to Device: ' + device['name'] + - ' on physical port ' + - str(device_link.get('south_facing_port')) + - ' to IP ' + device.get('ip') + - ':' + str(device.get('ip_port'))) - - if add_di: - sw.add_data_inspection(dev_id=device['id'], - dev_mac=device['mac']) - - def __process_gw_attack(self, attack, host): - attack_switch = None - for switch in self.switches: - if switch.type == 'gateway': - di_match_mac = switch.get_data_inspection_src_mac_keys() - if len(di_match_mac) > 0: - logger.debug( - 'Data inspection table keys on device [%s] - [%s]', - switch.sw_info['id'], di_match_mac) - if attack['src_mac'] in di_match_mac: - logger.info('Found source switch - [%s]', switch.name) - attack_switch = switch - else: - attack_switch = self.switches[0] - break - if attack_switch: - logger.info('Adding an attack [%s] to host [%s] and switch [%s]', - attack, host, attack_switch.name) - ip_addr = ipaddress.ip_address(attack['src_ip']) - logger.info('Attack ip addr - [%s]', ip_addr) - logger.debug('Attack ip addr class - [%s]', ip_addr.__class__) - if ip_addr.version == 6: - logger.debug('Attack is IPv6') - proto_key = 'ipv6' - else: - logger.debug('Attack is IPv4') - proto_key = 'ipv4' - - dst_addr_key = 'hdr.{}.dstAddr'.format(proto_key) - return attack_switch, dst_addr_key - else: - return None - - def add_attacker(self, attack, host): - logger.info('Attack received by the controller of type [%s] - [%s]', - self.switch_type, attack) - attack_switch, dst_addr_key = self.__process_gw_attack(attack, host) - if attack_switch and dst_addr_key: - attack['host'] = host - attack[dst_addr_key] = attack['src_ip'] - attack_switch.add_attack(**attack) - - def remove_attacker(self, attack, host): - attack_switch, dst_addr_key = self.__process_gw_attack(attack, host) - if attack_switch and dst_addr_key: - attack['host'] = host - attack[dst_addr_key] = attack['src_ip'] - attack_switch.stop_attack(**attack) - - def count_dropped_packets(self): - pass diff --git a/trans_sec/controller/http_server_flask.py b/trans_sec/controller/http_server_flask.py index 929e3337..17c82b9d 100644 --- a/trans_sec/controller/http_server_flask.py +++ b/trans_sec/controller/http_server_flask.py @@ -49,11 +49,6 @@ def start(self): DataInspection, '/dataInspection', resource_class_kwargs={'sdn_controller': self.sdn_controller}) - logger.info('Starting gwAttack') - self.api.add_resource( - GwAttack, '/gwAttack', - resource_class_kwargs={'sdn_controller': self.sdn_controller}) - logger.info('Starting aggAttack') self.api.add_resource( AggAttack, '/aggAttack', @@ -187,48 +182,6 @@ def delete(self): return json.dumps({"success": True}), 201 -class GwAttack(Resource): - """ - Class for exposing web service to issue an attack call to gateway.p4 - """ - - parser = reqparse.RequestParser() - parser.add_argument('src_mac', type=str) - parser.add_argument('src_ip', type=str) - parser.add_argument('dst_ip', type=str) - parser.add_argument('dst_port', type=str) - parser.add_argument('packet_size', type=str) - parser.add_argument('attack_type', type=str) - - def __init__(self, **kwargs): - logger.info('Starting GwAttack context') - self.sdn_controller = kwargs['sdn_controller'] - - @swagger.tags(['gatewayAttackStart']) - @swagger.response(response_code=201, - description='Mitigated attack on gateway') - @swagger.reqparser(name='GwAttackParser', parser=parser) - def post(self): - logger.info('Gateway attack requested') - args = self.parser.parse_args() - - logger.info('args - [%s]', args) - self.sdn_controller.add_attacker(args) - return json.dumps({"success": True}), 201 - - @swagger.tags(['gatewayAttackStop']) - @swagger.response(response_code=201, - description='Unmitigated attacks from gateway') - @swagger.reqparser(name='GwAttackParser', parser=parser) - def delete(self): - logger.info('GW Attacker to remove') - args = self.parser.parse_args() - - logger.info('args - [%s]', args) - self.sdn_controller.remove_attacker(args) - return json.dumps({"success": True}), 201 - - class AggDataForward(Resource): """ Class for exposing web service to enter a data_forward entry into the P4 diff --git a/trans_sec/device_software/hulk-attack.sh b/trans_sec/device_software/hulk-attack.sh deleted file mode 100755 index afe4b119..00000000 --- a/trans_sec/device_software/hulk-attack.sh +++ /dev/null @@ -1,24 +0,0 @@ - #!/bin/bash - -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -PY_SCRIPT=$1 -HOST=$2 -DURATION=$3 - -python $PY_SCRIPT http://${HOST} & -HULK_PID=$(echo $!) -sleep $DURATION -kill -9 $HULK_PID -echo "-- HULK Attack Terminated --" diff --git a/trans_sec/mininet/__init__.py b/trans_sec/mininet/__init__.py deleted file mode 100644 index 77c9835a..00000000 --- a/trans_sec/mininet/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/trans_sec/mininet/apptopo.py b/trans_sec/mininet/apptopo.py deleted file mode 100644 index b58e4d0d..00000000 --- a/trans_sec/mininet/apptopo.py +++ /dev/null @@ -1,89 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from mininet.topo import Topo - - -class AppTopo(Topo): - - def __init__(self, links, latencies, log_dir="/tmp", bws=None, **opts): - Topo.__init__(self, **opts) - - nodes = sum(map(list, zip(*links)), []) - host_names = sorted(list(set(filter(lambda n: n[0] == 'h', nodes)))) - sw_names = sorted(list(set(filter(lambda n: n[0] == 's', nodes)))) - sw_ports = dict([(sw, []) for sw in sw_names]) - - self.host_links = dict() - self.sw_links = dict([(sw, {}) for sw in sw_names]) - - for sw_name in sw_names: - self.addSwitch(sw_name, log_file="%s/%s.log" % (log_dir, sw_name)) - - for host_name in host_names: - host_num = int(host_name[1:]) - - self.addHost(host_name) - - self.host_links[host_name] = {} - host_links = filter( - lambda l: l[0] == host_name or l[1] == host_name, links) - - sw_idx = 0 - for link in host_links: - sw = link[0] if link[0] != host_name else link[1] - sw_num = int(sw[1:]) - assert sw[0] ==\ - 's', "Hosts should be connected to switches, "\ - "not {}".format(str(sw)) - host_ip = "10.0.%d.%d" % (sw_num, host_num) - host_mac = '00:00:00:00:%02x:%02x' % (sw_num, host_num) - delay_key = ''.join([host_name, sw]) - delay = latencies[ - delay_key] if delay_key in latencies else '0ms' - bw = bws[delay_key] if delay_key in bws else None - sw_ports[sw].append(host_name) - self.host_links[host_name][sw] = dict( - idx=sw_idx, - host_mac=host_mac, - host_ip=host_ip, - sw=sw, - sw_mac="00:00:00:00:%02x:%02x" % (sw_num, host_num), - sw_ip="10.0.%d.%d" % (sw_num, 254), - sw_port=sw_ports[sw].index(host_name) + 1 - ) - self.addLink(host_name, sw, delay=delay, bw=bw, - addr1=host_mac, - addr2=self.host_links[host_name][sw]['sw_mac']) - sw_idx += 1 - - for link in links: # only check switch-switch links - sw1, sw2 = link - if sw1[0] != 's' or sw2[0] != 's': - continue - - delay_key = ''.join(sorted([sw1, sw2])) - delay = latencies[delay_key] if delay_key in latencies else '0ms' - bw = bws[delay_key] if delay_key in bws else None - - self.addLink(sw1, sw2, delay=delay, bw=bw) # , max_queue_size=10) - sw_ports[sw1].append(sw2) - sw_ports[sw2].append(sw1) - - sw1_num, sw2_num = int(sw1[1:]), int(sw2[1:]) - sw1_port = dict(mac="00:00:00:%02x:%02x:00" % (sw1_num, sw2_num), - port=sw_ports[sw1].index(sw2) + 1) - sw2_port = dict(mac="00:00:00:%02x:%02x:00" % (sw2_num, sw1_num), - port=sw_ports[sw2].index(sw1) + 1) - - self.sw_links[sw1][sw2] = [sw1_port, sw2_port] - self.sw_links[sw2][sw1] = [sw2_port, sw1_port] diff --git a/trans_sec/mininet/exercise.py b/trans_sec/mininet/exercise.py deleted file mode 100644 index f0f5567e..00000000 --- a/trans_sec/mininet/exercise.py +++ /dev/null @@ -1,343 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Unit tests for convert.py -import logging -import os - -from mininet.cli import CLI -from mininet.link import TCLink, Intf -from mininet.net import Mininet -from mininet.topo import Topo - -from trans_sec.mininet.p4_mininet import P4Host -from trans_sec.p4runtime_lib.p4runtime_switch import MininetSwitch - -logger = logging.getLogger('exercise') - - -class ExerciseTopo(Topo): - """ - The mininet topology class for the P4 tutorial exercises. - A custom class is used because the exercises make a few topology - assumptions, mostly about the IP and MAC addresses. - """ - - def __init__(self, hosts, switches, external, links, log_dir, **opts): - Topo.__init__(self, **opts) - self.sw_port_mapping = {} - - for sw_name, sw in switches.items(): - dpid = self.__int_to_dpid(sw['id']) - self.addSwitch(sw['name'], dpid=dpid, - log_file="%s/%s.log" % (log_dir, sw)) - - for link in links: - np = link.get('north_facing_port') - sp = link.get('south_facing_port') - - # Two switches - if np and sp: - s_switch = switches.get(link.get('south_node')) - n_switch = switches.get(link.get('north_node')) - - self.addLink(s_switch['name'], n_switch['name'], - delay=link['latency'], bw=link['bandwidth'], - addr1=s_switch['mac'], - addr2=n_switch['mac']) - self.__add_switch_port(s_switch['name'], - n_switch['name'], - link['north_facing_port']) - self.__add_switch_port(n_switch['name'], - s_switch['name'], - link['south_facing_port']) - logger.info('Adding Switch Link %s %s port:%d <--> port:%d' % ( - link['south_node'], link['north_node'], - link['north_facing_port'], link['south_facing_port'])) - - # South switch has a north facing port - elif np: - s_switch = switches.get(link.get('south_node')) - n_host = hosts.get(link.get('north_node')) - if n_host is None: - n_host = external.get(link.get('north_node')) - # ignore externals - if n_host is not None: - self.addHost(n_host['name'], - ip=n_host['ip'] + '/24', - mac=n_host['mac']) - self.addLink(n_host['name'], s_switch['name'], - delay=link['latency'], bw=link['bandwidth'], - addr1=n_host['mac'], - addr2=s_switch['mac']) - self.__add_switch_port(s_switch['name'], - n_host['name'], np) - logger.info( - "Adding host %s link %s %s to switch %s %s on port %s", - n_host.get('name'), n_host.get('ip'), - n_host.get('mac'), s_switch.get('name'), - s_switch.get('mac'), np) - # North switch has a south facing port to the host - elif sp is not None: - n_switch = switches.get(link.get('north_node')) - s_host = hosts.get(link.get('south_node')) - - # ignore externals - if s_host is not None: - self.addHost(s_host['name'], - ip=s_host['ip'] + '/24', - mac=s_host['mac']) - if link.get('south_facing_mac'): - self.addLink(s_host['name'], n_switch['name'], - delay=link['latency'], - bw=link['bandwidth'], - addr1=s_host['mac'], - addr2=link.get('south_facing_mac')) - self.__add_switch_port(n_switch['name'], - s_host['name'], sp) - logger.info("Adding host %s link %s %s to switch %s %s" - " on port %d", - s_host.get('name'), s_host.get('ip'), - s_host.get('mac'), n_switch.get('name'), - link.get('south_facing_port'), sp,) - else: - self.addLink(s_host['name'], n_switch['name'], - delay=link['latency'], - bw=link['bandwidth'], - addr1=s_host['mac'], - addr2=n_switch['mac']) - self.__add_switch_port(n_switch['name'], - s_host['name'], sp) - logger.info("Adding host %s link %s %s to switch %s %s" - " on port %d", - s_host.get('name'), s_host.get('ip'), - s_host.get('mac'), n_switch.get('name'), - n_switch['mac'], sp,) - else: - logger.info('Error in link. At least one port must be ' - 'defined %s', link) - - self.__print_port_mapping() - - @staticmethod - def __int_to_dpid(dpid): - try: - dpid = hex(dpid)[2:] - dpid = '0' * (16 - len(dpid)) + dpid - return dpid - except IndexError: - raise Exception('Unable to derive default data path ID - ' - 'please either specify a dpid or use a ' - 'canonical switch name such as s23.') - - def __add_switch_port(self, sw, target, port=None): - if sw not in self.sw_port_mapping: - self.sw_port_mapping[sw] = [] - if port is None: - port_num = len(self.sw_port_mapping[sw]) + 1 - self.sw_port_mapping[sw].append((port_num, target)) - else: - self.sw_port_mapping[sw].append((port, target)) - - def __print_port_mapping(self): - logger.info("Switch port mapping:") - for sw in sorted(self.sw_port_mapping.keys()): - logger.info("%s: " % sw,) - for port_num, target in self.sw_port_mapping[sw]: - logger.info("%d:%s\t" % (port_num, target),) - - -class ExerciseRunner: - """ - Attributes: - log_dir : string // directory for mininet log files - pcap_dir : string // directory for mininet switch pcap files - hosts : list // list of mininet host names - switches : dict // mininet host names and their - associated properties - links : list // list of mininet link properties - switch_json : string // json of the compiled p4 example - topo : Topo object // The mininet topology instance - mininet : Mininet object // The mininet instance - """ - - def __init__(self, topo, log_dir, pcap_dir, switch_json, start_cli=False): - """ Initializes some attributes and reads the topology json. Does not - actually run the exercise. Use run_exercise() for that. - - Arguments: - topo : dict // A dict describing the mininet topology - log_dir : string // Path to a directory for storing - exercise logs - pcap_dir : string // Same for mininet switch pcap files - switch_json : string // Path to a compiled p4 json for bmv2 - """ - - logger.info('Instantiating P4 Mininet exercise with topology - [%s]', - topo) - self.hosts = topo['hosts'] - self.switches = topo['switches'] - self.external = topo.get('external') - self.links = topo['links'] - self.start_cli = start_cli - - # Ensure all the needed directories exist and are directories - for dir_name in [log_dir, pcap_dir]: - if dir_name and not os.path.isdir(dir_name): - if os.path.exists(dir_name): - raise Exception( - "'%s' exists and is not a directory!" % dir_name) - os.mkdir(dir_name) - self.log_dir = log_dir - self.pcap_dir = pcap_dir - self.switch_json = switch_json - self.topo = ExerciseTopo(self.hosts, self.switches, self.external, - self.links, self.log_dir) - self.mininet = self.__setup_mininet() - self.running = False - self.fwd_runner = None - - def run_exercise(self): - """ Sets up the mininet instance, programs the switches, - and starts the mininet CLI. This is the main method to run after - initializing the object. - """ - # Initialize mininet with the topology specified by the config - logger.info('Running exercise') - self.__add_external_connections() - self.mininet.start() - - logger.info('Programming mininet hosts') - self.__program_hosts() - - if self.start_cli: - logger.info('Starting mininet CLI') - self.do_net_cli() - - logger.info('Completed mininet setup') - self.running = True - - def stop(self): - if self.fwd_runner: - self.fwd_runner.stop() - self.running = False - - def __setup_mininet(self): - """ Create the mininet network object, and store it as self.mininet. - - Side effects: - - Mininet topology instance stored as self.topo - - Mininet instance stored as self.mininet - """ - switch_class = self.__configure_p4_switch() - - logger.info('Starting mininet with topology - [%s]', self.topo) - return Mininet( - topo=self.topo, link=TCLink, host=P4Host, switch=switch_class, - autoPinCpus=True) - - def __configure_p4_switch(self): - """ - Returns a P4RuntimeSwitch class for use by Mininet - """ - if self.pcap_dir: - pcap_dump_dir = self.pcap_dir - else: - pcap_dump_dir = self.log_dir - - switch_args = { - 'sw_path': 'simple_switch_grpc', - 'json_path': self.switch_json, - 'log_console': True, - 'pcap_dump': pcap_dump_dir, - } - - class ConfiguredMininetSwitch(MininetSwitch): - def __init__(self, *opts, **kwargs): - kwargs.update(switch_args) - MininetSwitch.__init__(self, *opts, **kwargs) - - def describe(self): - logger.info("%s -> gRPC port: %s", self.name, - self.grpc_port) - - return ConfiguredMininetSwitch - - def __add_external_connections(self): - if self.external: - logger.info('External dict contents - [%s]', self.external) - for link in self.links: - external = self.external.get(link['north_node']) - if external is not None: - sw_obj = self.mininet.get(link['south_node']) - Intf(external['id'], node=sw_obj) - - def __program_hosts(self): - logger.info('Programming hosts with values - [%s]', self.hosts) - for name, host in self.hosts.items(): - logger.debug('Programming host - [%s]', host) - h = self.mininet.get(host['name']) - h_iface = list(h.intfs.values())[0] - link = h_iface.link - - sw_iface = link.intf1 if link.intf1 != h_iface else link.intf2 - sw_ip = host['switch_ip'] - - # Ensure each host's interface name is unique, or else - # mininet cannot shutdown gracefully - h.defaultIntf().rename('%s-eth0' % host['name']) - # static arp entries and default routes - h.cmd('arp -i %s -s %s %s' % (h_iface.name, sw_ip, sw_iface.mac)) - h.cmd('ethtool --offload %s rx off tx off' % h_iface.name) - h.cmd('route add default gw %s %s' % (sw_ip, h_iface.name)) - h.cmd('/usr/sbin/sshd -D&') - h.setDefaultRoute("via %s" % sw_ip) - - def do_net_cli(self): - """ Starts up the mininet CLI and prints some helpful output. - - Assumes: - - A mininet instance is stored as self.mininet and - self.mininet.start() has been called. - """ - for s in self.mininet.switches: - s.describe() - for h in self.mininet.hosts: - try: - h.describe() - except Exception as e: - logger.warning("Ignore exception [%s]", e) - logger.info("Starting mininet CLI") - # Generate a message that will be printed by the Mininet CLI to make - # interacting with the simple switch a little easier. - print('') - print('==============================================================') - print('Welcome to the BMV2 Mininet CLI!') - print('==============================================================') - print('Your P4 program is installed into the BMV2 software switch') - print('and your initial configuration is loaded. You can interact') - print('with the network using the mininet CLI below.') - print('') - if self.switch_json: - print('To inspect or change the switch configuration, connect to') - print('CLI from your host operating system using this command:') - print(' simple_switch_CLI --thrift-port ') - print('') - print('To view a switch log, run this command from your host OS:') - print(' tail -f %s/.log' % self.log_dir) - print('') - print('To view the switch output pcap, check the pcap files in %s:' - % self.pcap_dir) - print(' for example run: sudo tcpdump -xxx -r s1-eth1.pcap') - print('') - - CLI(self.mininet) diff --git a/trans_sec/mininet/p4_mininet.py b/trans_sec/mininet/p4_mininet.py deleted file mode 100644 index 3c240f2b..00000000 --- a/trans_sec/mininet/p4_mininet.py +++ /dev/null @@ -1,159 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -import os -import tempfile -from sys import exit -from time import sleep - -from mininet.log import info, error, debug -from mininet.moduledeps import pathCheck -from mininet.node import Switch, Host - -from trans_sec.utils.netstat import check_listening_on_port - -SWITCH_START_TIMEOUT = 10 # seconds - -logger = logging.getLogger('p4_mininet') - - -class P4Host(Host): - def config(self, **params): - r = super(Host, self).config(**params) - - self.defaultIntf().rename("eth0") - - for off in ["rx", "tx", "sg"]: - cmd = "/sbin/ethtool --offload eth0 %s off" % off - self.cmd(cmd) - - # disable IPv6 - self.cmd("sysctl -w net.ipv6.conf.all.disable_ipv6=1") - self.cmd("sysctl -w net.ipv6.conf.default.disable_ipv6=1") - self.cmd("sysctl -w net.ipv6.conf.lo.disable_ipv6=1") - - return r - - def describe(self): - logger.info("Name: %s, dflt intf: %s/%s/%s", - self.name, - self.defaultIntf().name, - self.defaultIntf().IP(), - self.defaultIntf().MAC()) - - -class P4Switch(Switch): - """P4 virtual switch""" - device_id = 0 - - def __init__(self, name, sw_path=None, json_path=None, - thrift_port=None, - pcap_dump=False, - log_console=False, - verbose=False, - device_id=None, - enable_debugger=False, - **kwargs): - Switch.__init__(self, name, **kwargs) - logger.info('The other P4Switch') - assert sw_path - assert json_path - # make sure that the provided sw_path is valid - pathCheck(sw_path) - # make sure that the provided JSON file exists - if not os.path.isfile(json_path): - error("Invalid JSON file.\n") - exit(1) - self.sw_path = sw_path - self.json_path = json_path - self.verbose = verbose - logfile = "/tmp/p4s.{}.log".format(self.name) - self.output = open(logfile, 'w') - self.thrift_port = thrift_port - if check_listening_on_port(self.thrift_port): - error('%s cannot bind port %d because it is bound by another ' - 'process\n' % (self.name, self.thrift_port)) - exit(1) - self.pcap_dump = pcap_dump - self.enable_debugger = enable_debugger - self.log_console = log_console - if device_id is not None: - self.device_id = device_id - P4Switch.device_id = max(P4Switch.device_id, device_id) - else: - self.device_id = P4Switch.device_id - P4Switch.device_id += 1 - self.nanomsg = "ipc:///tmp/bm-{}-log.ipc".format(self.device_id) - - @classmethod - def setup(cls): - pass - - def check_switch_started(self, pid): - """While the process is running (pid exists), we check if the Thrift - server has been started. If the Thrift server is ready, we assume that - the switch was started successfully. This is only reliable if the - Thrift server is started at the end of the init process""" - while True: - if not os.path.exists(os.path.join("/proc", str(pid))): - return False - if check_listening_on_port(self.thrift_port): - return True - sleep(0.5) - - def start(self, controllers): - """ - Start up a new P4 switch - """ - info("Starting P4 switch {}.\n".format(self.name)) - args = [self.sw_path] - for port, intf in self.intfs.items(): - if not intf.IP(): - args.extend(['-i', str(port) + "@" + intf.name]) - if self.pcap_dump: - args.append("--pcap pcaps") - # args.append("--useFiles") - if self.thrift_port: - args.extend(['--thrift-port', str(self.thrift_port)]) - if self.nanomsg: - args.extend(['--nanolog', self.nanomsg]) - args.extend(['--device-id', str(self.device_id)]) - P4Switch.device_id += 1 - args.append(self.json_path) - if self.enable_debugger: - args.append("--debugger") - if self.log_console: - args.append("--log-console") - logfile = "/tmp/p4s.{}.log".format(self.name) - info(' '.join(args) + "\n") - - with tempfile.NamedTemporaryFile() as f: - self.cmd(' '.join( - args) + ' >' + logfile + ' 2>&1 & echo $! >> ' + f.name) - pid = int(f.read()) - debug("P4 switch {} PID is {}.\n".format(self.name, pid)) - if not self.check_switch_started(pid): - error("P4 switch {} did not start correctly.\n".format(self.name)) - exit(1) - info("P4 switch {} has been started.\n".format(self.name)) - - def stop(self, delete_interfaces=True): - """ - Terminate P4 switch. - """ - self.output.flush() - self.cmd('kill %' + self.sw_path) - self.cmd('wait') - self.deleteIntfs() - super(P4Switch, self).stop(delete_interfaces) diff --git a/trans_sec/p4runtime_lib/__init__.py b/trans_sec/p4runtime_lib/__init__.py deleted file mode 100644 index 77c9835a..00000000 --- a/trans_sec/p4runtime_lib/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/trans_sec/p4runtime_lib/aggregate_switch.py b/trans_sec/p4runtime_lib/aggregate_switch.py deleted file mode 100644 index f41d2601..00000000 --- a/trans_sec/p4runtime_lib/aggregate_switch.py +++ /dev/null @@ -1,148 +0,0 @@ -# Copyright (c) 2020 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Originally copied from: -# -# Copyright 2017-present Open Networking Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import logging - -from trans_sec.p4runtime_lib.p4rt_switch import P4RuntimeSwitch -from trans_sec.consts import UDP_INT_DST_PORT -from trans_sec.controller.ddos_sdn_controller import AGG_CTRL_KEY - -logger = logging.getLogger('aggregate_switch') - - -class AggregateSwitch(P4RuntimeSwitch): - def __init__(self, sw_info, proto_dump_file=None): - """ - Construct Switch class to control BMV2 switches running gateway.p4 - """ - super(self.__class__, self).__init__( - sw_info, 'TpsAggIngress', 'TpsEgress', proto_dump_file) - - def write_multicast_entry(self, hosts): - super(self.__class__, self).write_multicast_entry(hosts) - self.write_arp_flood() - - def add_data_inspection(self, dev_id, dev_mac): - logger.info( - 'Adding data inspection to aggregate device [%s] with device ID ' - '- [%s] and mac - [%s]', self.device_id, dev_id, dev_mac) - # Northbound Traffic Inspection for IPv4 - action_params = { - 'device': dev_id, - 'switch_id': self.int_device_id - } - table_entry = self.p4info_helper.build_table_entry( - table_name='{}.data_inspection_t'.format(self.p4_ingress), - match_fields={ - 'hdr.ethernet.src_mac': dev_mac, - }, - action_name='{}.data_inspect_packet'.format( - self.p4_ingress), - action_params=action_params - ) - self.write_table_entry(table_entry) - - logger.info( - 'Installed Northbound Packet Inspection for device - [%s]' - ' with MAC - [%s] with action params - [%s]', - AGG_CTRL_KEY, dev_mac, action_params) - - def del_data_inspection(self, dev_id, dev_mac): - logger.info( - 'Adding data inspection to aggregate device [%s] with device ID ' - '- [%s] and mac - [%s]', self.device_id, dev_id, dev_mac) - # Northbound Traffic Inspection for IPv4 - action_params = { - 'device': dev_id, - 'switch_id': self.int_device_id - } - table_entry = self.p4info_helper.build_table_entry( - table_name='{}.data_inspection_t'.format(self.p4_ingress), - match_fields={ - 'hdr.ethernet.src_mac': dev_mac, - }, - action_name='{}.data_inspect_packet'.format( - self.p4_ingress) - ) - self.delete_table_entry(table_entry) - - logger.info( - 'Installed Northbound Packet Inspection for device - [%s]' - ' with MAC - [%s] with action params - [%s]', - AGG_CTRL_KEY, dev_mac, action_params) - - def add_attack(self, **kwargs): - logger.info('Adding attack [%s]', kwargs) - action_name, dst_ipv4, dst_ipv6 = self.parse_attack(**kwargs) - self.insert_p4_table_entry( - table_name='data_drop_t', - action_name=action_name, - match_fields={ - 'hdr.ethernet.src_mac': kwargs['src_mac'], - 'meta.ipv4_addr': dst_ipv4, - 'meta.ipv6_addr': dst_ipv6, - 'meta.dst_port': int(kwargs['dst_port']), - }, - action_params=None, - ingress_class=True, - ) - logger.info('%s Dropping TCP Packets from %s', - self.name, kwargs.get('src_ip')) - - def stop_attack(self, **kwargs): - logger.info('Adding attack [%s]', kwargs) - action_name, dst_ipv4, dst_ipv6 = self.parse_attack(**kwargs) - - self.delete_p4_table_entry( - table_name='data_drop_t', - action_name=action_name, - match_fields={ - 'hdr.ethernet.src_mac': kwargs['src_mac'], - 'meta.ipv4_addr': dst_ipv4, - 'meta.ipv6_addr': dst_ipv6, - 'meta.dst_port': int(kwargs['dst_port']), - }, - ingress_class=True, - ) - logger.info('%s Dropping TCP Packets from %s', - self.name, kwargs.get('src_ip')) - - def add_switch_id(self): - action_params = { - 'switch_id': self.int_device_id - } - table_entry = self.p4info_helper.build_table_entry( - table_name='{}.add_switch_id_t'.format(self.p4_ingress), - match_fields={ - 'hdr.udp_int.dst_port': UDP_INT_DST_PORT - }, - action_name='{}.add_switch_id'.format( - self.p4_ingress), - action_params=action_params) - self.write_table_entry(table_entry) diff --git a/trans_sec/p4runtime_lib/core_switch.py b/trans_sec/p4runtime_lib/core_switch.py deleted file mode 100644 index 9f063e1f..00000000 --- a/trans_sec/p4runtime_lib/core_switch.py +++ /dev/null @@ -1,143 +0,0 @@ -# Copyright (c) 2020 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Originally copied from: -# -# Copyright 2017-present Open Networking Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import logging -import socket - -from trans_sec.consts import UDP_INT_DST_PORT -from trans_sec.p4runtime_lib.p4rt_switch import P4RuntimeSwitch - -logger = logging.getLogger('core_switch') - - -class CoreSwitch(P4RuntimeSwitch): - def __init__(self, sw_info, proto_dump_file=None): - """ - Construct Switch class to control BMV2 switches running gateway.p4 - """ - super(self.__class__, self).__init__( - sw_info, 'TpsCoreIngress', 'TpsCoreEgress', proto_dump_file) - - def write_multicast_entry(self, hosts): - try: - super(self.__class__, self).write_multicast_entry(hosts) - except Exception as e: - logger.warning('Unexpected error writing multicast entry - [%s]', - e) - self.write_arp_flood() - - def add_data_forward(self, dst_mac, egress_port): - logger.info( - 'Adding data forward to core device [%s] with source_mac ' - '- [%s] and ingress port - [%s]', - self.device_id, dst_mac, egress_port) - inserted = super(self.__class__, self).add_data_forward( - dst_mac, egress_port) - - if inserted: - table_entry = self.p4info_helper.build_table_entry( - table_name='{}.arp_forward_t'.format(self.p4_ingress), - match_fields={ - 'hdr.ethernet.dst_mac': dst_mac - }, - action_name='{}.arp_forward'.format(self.p4_ingress), - action_params={'port': egress_port} - ) - self.write_table_entry(table_entry) - - def add_data_inspection(self, dev_id, dev_mac): - logger.info( - 'Adding data inspection entry to core device [%s] with device ID ' - '- [%s]', self.int_device_id, dev_id) - - action_params = { - 'switch_id': self.int_device_id - } - table_name = '{}.data_inspection_t'.format(self.p4_ingress) - action_name = '{}.data_inspect_packet'.format(self.p4_ingress) - logger.info( - 'Insert params into table - [%s] for action [%s] ' - 'with params [%s]', - table_name, action_name, action_params) - table_entry = self.p4info_helper.build_table_entry( - table_name=table_name, - match_fields={ - 'hdr.udp_int.dst_port': UDP_INT_DST_PORT - }, - action_name=action_name, - action_params=action_params - ) - self.write_table_entry(table_entry) - - def del_data_inspection(self, dev_id, dev_mac): - logger.info( - 'Adding data inspection entry to core device [%s] with device ID ' - '- [%s]', self.device_id, dev_id) - - action_params = { - 'switch_id': dev_id - } - table_name = '{}.data_inspection_t'.format(self.p4_ingress) - action_name = '{}.data_inspect_packet'.format(self.p4_ingress) - logger.info( - 'Insert params into table - [%s] for action [%s] ' - 'with params [%s]', - table_name, action_name, action_params) - table_entry = self.p4info_helper.build_table_entry( - table_name=table_name, - match_fields={ - 'hdr.udp_int.dst_port': UDP_INT_DST_PORT - }, - action_name=action_name, - ) - self.delete_table_entry(table_entry) - - def setup_telemetry_rpt(self, ae_ip): - logger.info( - 'Setting up telemetry report on core device [%s] with ' - 'AE IP - [%s]', self.device_id, ae_ip) - - ae_ip_addr = socket.gethostbyname(ae_ip) - logger.info( - 'Starting telemetry report for INT headers with dst_port ' - 'value of 555 to AE IP [%s]', ae_ip_addr) - table_name = '{}.setup_telemetry_rpt_t'.format(self.p4_egress) - action_name = '{}.setup_telem_rpt_ipv4'.format(self.p4_egress) - match_fields = { - 'hdr.udp_int.dst_port': UDP_INT_DST_PORT - } - action_params = { - 'ae_ip': ae_ip_addr - } - table_entry = self.p4info_helper.build_table_entry( - table_name=table_name, - match_fields=match_fields, - action_name=action_name, - action_params=action_params) - self.write_table_entry(table_entry) diff --git a/trans_sec/p4runtime_lib/gateway_switch.py b/trans_sec/p4runtime_lib/gateway_switch.py deleted file mode 100644 index 064db846..00000000 --- a/trans_sec/p4runtime_lib/gateway_switch.py +++ /dev/null @@ -1,349 +0,0 @@ -# Copyright (c) 2020 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Originally copied from: -# -# Copyright 2017-present Open Networking Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import ipaddress -import logging -from abc import ABC - -from trans_sec.p4runtime_lib.p4rt_switch import P4RuntimeSwitch -from trans_sec.utils.convert import decode_num, decode_ipv4 - -logger = logging.getLogger('gateway_switch') - - -class GatewaySwitch(P4RuntimeSwitch, ABC): - def __init__(self, sw_info, proto_dump_file=None): - """ - Construct Switch class to control BMV2 switches running gateway.p4 - """ - super(self.__class__, self).__init__( - sw_info, 'TpsGwIngress', 'TpsEgress', proto_dump_file) - self.nat_udp_ports = set() - self.nat_tcp_ports = set() - self.tcp_port_count = 1 - self.udp_port_count = 1 - - def start_digest_listeners(self): - logger.debug('Starting digest listener for - [%s]', self.sw_info) - if 'arch' in self.sw_info and self.sw_info.get('arch') == 'tofino': - logger.info('Tofino currently not supporting digests') - pass - else: - logger.info('Building digest entry') - digest_entry, digest_info = self.p4info_helper.build_digest_entry( - digest_name="nat_digest") - self.write_digest_entry(digest_entry) - super(self.__class__, self).start_digest_listeners() - - def receive_nat_digests(self): - """ - Runnable method for self.digest_thread - """ - logger.info("Started listening NAT digest thread for %s", - self.name) - while True: - try: - logger.debug('Requesting NAT digests') - digests = self.digest_list() - digest_data = digests.digest.data - self.interpret_nat_digest(digest_data) - logger.debug('Interpreted NAT digest data') - except Exception as e: - logger.error( - 'Unexpected error reading NAT digest from [%s] - [%s]', - self.name, e) - - def add_data_inspection(self, dev_id, dev_mac): - logger.info( - 'Adding data inspection to gateway device [%s] with device ID ' - '- [%s] and mac - [%s]', self.device_id, dev_id, dev_mac) - # Northbound Traffic Inspection for IPv4 - action_params = { - 'device': dev_id, - 'switch_id': self.device_id - } - table_entry = self.p4info_helper.build_table_entry( - table_name='{}.data_inspection_t'.format(self.p4_ingress), - match_fields={ - 'hdr.ethernet.src_mac': dev_mac, - }, - action_name='{}.data_inspect_packet'.format( - self.p4_ingress), - action_params=action_params - ) - self.write_table_entry(table_entry) - - logger.info( - 'Installed Northbound Packet Inspection for device with' - ' MAC - [%s] with action params - [%s]', - dev_mac, action_params) - - def del_data_inspection(self, dev_id, dev_mac): - logger.info( - 'Adding data inspection to gateway device [%s] with device ID ' - '- [%s] and mac - [%s]', self.device_id, dev_id, dev_mac) - # Northbound Traffic Inspection for IPv4 - action_params = { - 'device': dev_id, - 'switch_id': self.device_id - } - table_entry = self.p4info_helper.build_table_entry( - table_name='{}.data_inspection_t'.format(self.p4_ingress), - match_fields={ - 'hdr.ethernet.src_mac': dev_mac, - }, - action_name='{}.data_inspect_packet'.format( - self.p4_ingress), - ) - self.delete_table_entry(table_entry) - - logger.info( - 'Installed Northbound Packet Inspection for device with' - ' MAC - [%s] with action params - [%s]', - dev_mac, action_params) - - @staticmethod - def parse_attack(**kwargs): - src_ip = ipaddress.ip_address(kwargs['src_ip']) - dst_ip = ipaddress.ip_address(kwargs['dst_ip']) - udp_table_name = 'data_drop_udp_ipv{}_t'.format(dst_ip.version) - tcp_table_name = 'data_drop_tcp_ipv{}_t'.format(dst_ip.version) - action_name = 'data_drop' - - logger.info('Attack src_ip - [%s], dst_ip - [%s]', src_ip, dst_ip) - # TODO - Add back source IP address as a match field after adding - # mitigation at the Aggregate - if dst_ip.version == 6: - logger.debug('Attack is IPv6') - proto_key = 'ipv6' - else: - logger.debug('Attack is IPv4') - proto_key = 'ipv4' - - dst_addr_key = 'hdr.{}.dstAddr'.format(proto_key) - - out = (udp_table_name, tcp_table_name, action_name, - str(dst_ip.exploded), dst_addr_key) - logger.info('Attack data - [%s]', out) - return out - - def add_attack(self, **kwargs): - logger.info('Adding attack [%s]', kwargs) - udp_tn, tcp_tn, action_name, dst_ip, dst_addr_key = \ - self.parse_attack(**kwargs) - - self.insert_p4_table_entry( - table_name=udp_tn, - action_name=action_name, - match_fields={ - 'hdr.ethernet.src_mac': kwargs['src_mac'], - dst_addr_key: str(dst_ip), - 'hdr.udp.dst_port': int(kwargs['dst_port']), - }, - action_params={'device': self.device_id}, - ingress_class=True, - ) - logger.info('%s Dropping UDP Packets from %s', - self.name, kwargs.get('src_ip')) - self.insert_p4_table_entry( - table_name=tcp_tn, - action_name=action_name, - match_fields={ - 'hdr.ethernet.src_mac': kwargs['src_mac'], - dst_addr_key: str(dst_ip), - 'hdr.tcp.dst_port': int(kwargs['dst_port']), - }, - action_params={'device': self.device_id}, - ingress_class=True, - ) - logger.info('%s Dropping TCP Packets from %s', - self.name, kwargs.get('src_ip')) - - def stop_attack(self, **kwargs): - logger.info('Stopping attack [%s]', kwargs) - udp_tn, tcp_tn, action_name, dst_ip, dst_addr_key = \ - self.parse_attack(**kwargs) - - self.delete_p4_table_entry( - table_name=udp_tn, - action_name=action_name, - match_fields={ - 'hdr.ethernet.src_mac': kwargs['src_mac'], - dst_addr_key: dst_ip, - 'hdr.udp.dst_port': int(kwargs['dst_port']), - }, - ingress_class=True, - ) - logger.info('%s no longer dropping UDP Packets from %s', - self.name, kwargs.get('src_ip')) - self.delete_p4_table_entry( - table_name=tcp_tn, - action_name=action_name, - match_fields={ - 'hdr.ethernet.src_mac': kwargs['src_mac'], - dst_addr_key: dst_ip, - 'hdr.tcp.dst_port': int(kwargs['dst_port']), - }, - ingress_class=True, - ) - logger.info('%s no longer dropping TCP Packets from %s', - self.name, kwargs.get('src_ip')) - - def add_nat_table(self, udp_source_port, tcp_source_port, source_ip): - gateway_public_ip = self.sw_info['public_ip'] - logger.info("Adding nat table entries on gateway device [%s] for %s", - self.device_id, source_ip) - logger.info("Check if %s not in %s for %s", udp_source_port, - self.nat_udp_ports, self.name) - # NAT Table Entries to handle UDP packets - if udp_source_port and udp_source_port not in self.nat_udp_ports: - table_entry = self.p4info_helper.build_table_entry( - table_name='{}.udp_local_to_global_t'.format(self.p4_ingress), - match_fields={ - 'hdr.udp.src_port': udp_source_port, - 'hdr.ipv4.srcAddr': (source_ip, 32) - }, - action_name='{}.udp_local_to_global'.format(self.p4_ingress), - action_params={ - 'src_port': int("50" + str(self.device_id) + str( - self.udp_port_count)), - 'ip_srcAddr': gateway_public_ip - }) - self.write_table_entry(table_entry) - table_entry = self.p4info_helper.build_table_entry( - table_name='{}.udp_global_to_local_t'.format(self.p4_ingress), - match_fields={ - 'hdr.udp.dst_port': int( - "50" + str(self.device_id) + str( - self.udp_port_count)), - 'hdr.ipv4.dstAddr': (gateway_public_ip, 32) - }, - action_name='{}.udp_global_to_local'.format(self.p4_ingress), - action_params={ - 'dst_port': udp_source_port, - 'ip_dstAddr': source_ip - }) - self.write_table_entry(table_entry) - self.udp_port_count = self.udp_port_count + 1 - self.nat_udp_ports.add(udp_source_port) - logger.info("UDP NAT table entry added on %s", - self.name) - elif tcp_source_port and tcp_source_port not in self.nat_tcp_ports: - # NAT Table Entries to handle TCP packets - table_entry = self.p4info_helper.build_table_entry( - table_name='{}.tcp_local_to_global_t'.format(self.p4_ingress), - match_fields={ - 'hdr.tcp.src_port': tcp_source_port, - 'hdr.ipv4.srcAddr': (source_ip, 32) - }, - action_name='{}.tcp_local_to_global'.format(self.p4_ingress), - action_params={ - 'src_port': int("50" + str(self.device_id) + str( - self.tcp_port_count)), - 'ip_srcAddr': gateway_public_ip - }) - self.write_table_entry(table_entry) - table_entry = self.p4info_helper.build_table_entry( - table_name='{}.tcp_global_to_local_t'.format(self.p4_ingress), - match_fields={ - 'hdr.tcp.dst_port': int( - "50" + str(self.device_id) + str( - self.tcp_port_count)), - 'hdr.ipv4.dstAddr': (gateway_public_ip, 32) - }, - action_name='{}.tcp_global_to_local'.format(self.p4_ingress), - action_params={ - 'dst_port': tcp_source_port, - 'ip_dstAddr': source_ip - }) - self.write_table_entry(table_entry) - self.tcp_port_count = self.tcp_port_count + 1 - self.nat_tcp_ports.add(tcp_source_port) - logger.info("TCP NAT table entry added on %s", - self.name) - - def interpret_nat_digest(self, digest_data): - logger.debug("Digest data %s", digest_data) - for members in digest_data: - logger.debug("Members: %s", members) - if members.WhichOneof('data') == 'struct': - udp_source_port = decode_num( - members.struct.members[0].bitstring) - logger.info('Learned UDP Source Port from %s is: %s', - self.name, udp_source_port) - tcp_source_port = decode_num( - members.struct.members[1].bitstring) - logger.info('Learned TCP Source Port from %s is: %s', - self.name, tcp_source_port) - source_ip = decode_ipv4(members.struct.members[2].bitstring) - logger.info('Learned Source IP Address from %s is: %s', - self.name, source_ip) - self.add_nat_table(udp_source_port, tcp_source_port, source_ip) - - def write_multicast_entry(self, hosts): - logger.debug('Writing multicast entries on gateway device [%s]', - self.device_id) - super(self.__class__, self).write_multicast_entry(hosts) - - # TODO/FIXME - We need to define something in the topology for - # determining which is the NB server as this will break outside of - # the tested topologies and probably has other unintended consequences - target_host = None - - logger.debug('Switch Hosts defs - [%s]', hosts) - - for host in hosts.values(): - logger.debug('Switch Host def - [%s]', host) - if host['type'] == 'target-server': - target_host = host - - if target_host: - table_entry = self.p4info_helper.build_table_entry( - table_name='{}.mac_lookup_ipv4_t'.format(self.p4_ingress), - match_fields={ - 'hdr.ipv4.dstAddr': (target_host['ip'], 32) - }, - action_name='{}.mac_lookup'.format(self.p4_ingress), - action_params={ - 'dst_mac': target_host['mac'] - }) - self.write_table_entry(table_entry) - table_entry = self.p4info_helper.build_table_entry( - table_name='{}.mac_lookup_ipv6_t'.format(self.p4_ingress), - match_fields={ - 'hdr.ipv6.dstAddr': (target_host['ipv6'], 128) - }, - action_name='{}.mac_lookup'.format(self.p4_ingress), - action_params={ - 'dst_mac': target_host['mac'] - }) - self.write_table_entry(table_entry) - else: - logger.warning('Target host not found, not setting the ' - 'multicast group') diff --git a/trans_sec/p4runtime_lib/helper.py b/trans_sec/p4runtime_lib/helper.py deleted file mode 100644 index f0fe12af..00000000 --- a/trans_sec/p4runtime_lib/helper.py +++ /dev/null @@ -1,297 +0,0 @@ -# Copyright (c) 2019 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Originally copied from: -# -# Copyright 2017-present Open Networking Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import logging -import re - -import google.protobuf.text_format -from p4.config.v1 import p4info_pb2 -from p4.v1 import p4runtime_pb2 - -from trans_sec.utils.convert import encode - -logger = logging.getLogger('helper') - - -class P4InfoHelper(object): - def __init__(self, p4_info_filepath): - p4info = p4info_pb2.P4Info() - # Load the p4info file into a skeleton P4Info object - with open(p4_info_filepath) as p4info_f: - google.protobuf.text_format.Merge(p4info_f.read(), p4info) - self.p4info = p4info - - def get(self, entity_type, name=None, entity_id=None): - if name is not None and entity_id is not None: - raise AssertionError("name or id must be None") - - for o in getattr(self.p4info, entity_type): - pre = o.preamble - if name: - if pre.name == name or pre.alias == name: - return o - else: - if pre.id == entity_id: - return o - - if name: - raise AttributeError( - "Could not find %r of type %s" % (name, entity_type)) - else: - raise AttributeError( - "Could not find id %r of type %s" % (entity_id, entity_type)) - - def get_id(self, entity_type, name): - return self.get(entity_type, name=name).preamble.id - - def get_name(self, entity_type, entity_id): - return self.get(entity_type, entity_id=entity_id).preamble.name - - def get_alias(self, entity_type, entity_id): - return self.get(entity_type, entity_id=entity_id).preamble.alias - - def __getattr__(self, attr): - # Synthesize convenience functions for name to id lookups for top-level - # entities - # e.g. get_tables_id(name_string) or get_actions_id(name_string) - m = re.search(r"^get_(\w+)_id$", attr) - if m: - primitive = m.group(1) - return lambda name: self.get_id(primitive, name) - - # Synthesize convenience functions for id to name lookups - # e.g. get_tables_name(id) or get_actions_name(id) - m = re.search(r"^get_(\w+)_name$", attr) - if m: - primitive = m.group(1) - return lambda prim_id: self.get_name(primitive, prim_id) - - raise AttributeError( - "%r object has no attribute %r" % (self.__class__, attr)) - - def get_match_field(self, table_name, name=None, field_id=None): - for t in self.p4info.tables: - pre = t.preamble - if pre.name == table_name: - for mf in t.match_fields: - if name is not None: - if mf.name == name: - return mf - elif field_id is not None: - if mf.id == field_id: - return mf - raise AttributeError("%r has no attribute %r" % ( - table_name, name if name is not None else field_id)) - - def get_match_field_id(self, table_name, match_field_name): - return self.get_match_field(table_name, name=match_field_name).id - - def get_match_field_name(self, table_name, match_field_id): - return self.get_match_field(table_name, field_id=match_field_id).name - - def get_match_field_pb(self, table_name, match_field_name, value): - logger.info( - 'Retrieving match field on table name - [%s], match field - [%s], ' - 'value - [%s]', table_name, match_field_name, value) - - p4info_match = self.get_match_field(table_name, match_field_name) - bit_width = p4info_match.bitwidth - p4runtime_match = p4runtime_pb2.FieldMatch() - p4runtime_match.field_id = p4info_match.id - match_type = p4info_match.match_type - - logger.info( - 'Encoding value [%s] for exact match with bitwidth - [%s]', - value, bit_width) - - if match_type == p4info_pb2.MatchField.EXACT: - logger.info('Encoding for EXACT matches') - exact = p4runtime_match.exact - if isinstance(value, list) or isinstance(value, tuple): - exact.value = encode(value[0], bit_width) - else: - exact.value = encode(value, bit_width) - elif match_type == p4info_pb2.MatchField.LPM: - logger.info('Encoding for LPM matches') - lpm = p4runtime_match.lpm - lpm.value = encode(value[0], bit_width) - lpm.prefix_len = value[1] - elif match_type == p4info_pb2.MatchField.TERNARY: - logger.info('Encoding for TERNARY matches') - ternary = p4runtime_match.ternary - ternary.value = encode(value[0], bit_width) - ternary.mask = encode(value[1], bit_width) - elif match_type == p4info_pb2.MatchField.RANGE: - logger.info('Encoding for RANGE matches') - range_match = p4runtime_match.range - range_match.low = encode(value[0], bit_width) - range_match.high = encode(value[1], bit_width) - else: - raise Exception("Unsupported match type with type %r" % match_type) - return p4runtime_match - - @staticmethod - def get_match_field_value(match_field): - match_type = match_field.WhichOneof("field_match_type") - if match_type == 'valid': - return match_field.valid.value - elif match_type == 'exact': - return match_field.exact.value - elif match_type == 'lpm': - return match_field.lpm.value, match_field.lpm.prefix_len - elif match_type == 'ternary': - return match_field.ternary.value, match_field.ternary.mask - elif match_type == 'range': - return match_field.range.low, match_field.range.high - else: - raise Exception("Unsupported match type with type %r" % match_type) - - def get_action_param(self, action_name, name=None, action_id=None): - action = None - for action in self.p4info.actions: - pre = action.preamble - if pre.name == action_name: - for param in action.params: - if name is not None: - if param.name == name: - return param - elif action_id is not None: - if param.id == action_id: - return param - if action: - raise AttributeError("action %r has no param %r, (has: %r)" % ( - action_name, name if name is not None else action_id, - action.params)) - else: - raise AttributeError("action %r has no param %r" % ( - action_name, name if name is not None else action_id)) - - def get_action_param_id(self, action_name, param_name): - return self.get_action_param(action_name, name=param_name).id - - def get_action_param_name(self, action_name, param_id): - return self.get_action_param(action_name, action_id=param_id).name - - def get_action_param_pb(self, action_name, param_name, value): - logger.info( - 'Retrieving action param for action - [%s], param - [%s], ' - 'value - [%s]', action_name, param_name, value) - p4info_param = self.get_action_param(action_name, param_name) - p4runtime_param = p4runtime_pb2.Action.Param() - p4runtime_param.param_id = p4info_param.id - p4runtime_param.value = encode(value, p4info_param.bitwidth) - return p4runtime_param - - def build_table_entry(self, table_name, match_fields=None, - default_action=False, action_name=None, - action_params=None, priority=None): - logger.info( - 'Building table entry to table [%s] with match_fields - [%s] ' - 'action - [%s] and params - [%s]', - table_name, match_fields, action_name, action_params) - table_entry = p4runtime_pb2.TableEntry() - table_entry.table_id = self.get_tables_id(table_name) - logger.debug('Table ID - [%s]', table_entry.table_id) - - if priority is not None: - table_entry.priority = priority - - if match_fields: - table_entry.match.extend([ - self.get_match_field_pb(table_name, match_field_name, value) - for match_field_name, value in match_fields.items() - ]) - - if default_action: - table_entry.is_default_action = True - - if action_name: - action = table_entry.action.action - action.action_id = self.get_actions_id(action_name) - if action_params: - logger.info('Action params - [%s]', action_params) - action.params.extend([ - self.get_action_param_pb(action_name, field_name, value) - for field_name, value in action_params.items() - ]) - return table_entry - - @staticmethod - def reset_counter(counter_id, index): - logger.info('Resetting counter with ID - [%s] and index - [%s]', - counter_id, index) - counter_entry = p4runtime_pb2.CounterEntry() - if counter_id is not None: - counter_entry.counter_id = counter_id - else: - counter_entry.counter_id = 0 - if index is not None: - counter_entry.index.index = index - counter_entry.data.byte_count = 0 - counter_entry.data.packet_count = 0 - - @staticmethod - def build_clone_entry(clone_egress): - logger.info('Building clone entry with egress_port value of - [%s]', - clone_egress) - pre_entry = p4runtime_pb2.PacketReplicationEngineEntry() - clone_session_entry = p4runtime_pb2.CloneSessionEntry() - clone_session_entry.session_id = 5 - clone_session_entry.replicas.add(egress_port=clone_egress, instance=1) - pre_entry.clone_session_entry.CopyFrom(clone_session_entry) - logger.info('Clone entry - [%s]', pre_entry) - return pre_entry - - def build_digest_entry(self, digest_name): - digest_info = {} - logger.info("Building digest entry for %s", digest_name) - digest_entry = p4runtime_pb2.DigestEntry() - # using name - # TODO/FIXME - This appears to be broken on Tofino see issue #175 - logger.debug('Before getting digest ID') - digest_entry.digest_id = self.get_digests_id(digest_name) - logger.debug('After getting digest ID') - # using id directly - digest_entry.config.max_timeout_ns = 0 - digest_entry.config.max_list_size = 1 - digest_entry.config.ack_timeout_ns = 0 - digest_info[digest_name] = digest_entry.digest_id - logger.info("Digest Entry Information %s", digest_info) - return digest_entry, digest_info - - @staticmethod - def build_multicast_group_entry(mc_group_id, replicas): - mc_entry = p4runtime_pb2.PacketReplicationEngineEntry() - mc_entry.multicast_group_entry.multicast_group_id = mc_group_id - for replica in replicas: - r = p4runtime_pb2.Replica() - r.egress_port = int(replica['egress_port']) - r.instance = int(replica['instance']) - mc_entry.multicast_group_entry.replicas.extend([r]) - return mc_entry diff --git a/trans_sec/p4runtime_lib/p4rt_switch.py b/trans_sec/p4runtime_lib/p4rt_switch.py deleted file mode 100644 index dea1a3ef..00000000 --- a/trans_sec/p4runtime_lib/p4rt_switch.py +++ /dev/null @@ -1,624 +0,0 @@ -# Copyright (c) 2020 Cable Television Laboratories, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import codecs -import logging -import struct -from abc import ABC -from threading import Thread - -import grpc -from p4.tmp import p4config_pb2 -from p4.v1 import p4runtime_pb2, p4runtime_pb2_grpc - -from trans_sec.controller.ddos_sdn_controller import GATEWAY_CTRL_KEY -from trans_sec.p4runtime_lib import helper -from trans_sec.switch import SwitchConnection, IterableQueue, GrpcRequestLogger -from trans_sec.utils.convert import decode_mac - -MSG_LOG_MAX_LEN = 1024 - -logger = logging.getLogger('switch') - - -class P4RuntimeSwitch(SwitchConnection, ABC): - - def __init__(self, sw_info, p4_ingress, p4_egress, proto_dump_file=None): - super(P4RuntimeSwitch, self).__init__(sw_info) - p4info_txt = sw_info['runtime_p4info'] - logger.info('Loading p4info_helper with file - [%s]', - p4info_txt) - self.p4info_helper = helper.P4InfoHelper(p4info_txt) - self.p4_ingress = p4_ingress - self.p4_egress = p4_egress - grpc_channel = grpc.insecure_channel(self.grpc_addr) - if proto_dump_file is not None: - logger.info('Adding interceptor with file - [%s] to device [%s]', - proto_dump_file, self.grpc_addr) - interceptor = GrpcRequestLogger(proto_dump_file) - grpc_channel = grpc.intercept_channel(grpc_channel, interceptor) - - logger.info('Creating client stub to channel - [%s] at address - [%s]', - grpc_channel, self.grpc_addr) - self.client_stub = p4runtime_pb2_grpc.P4RuntimeStub(grpc_channel) - self.requests_stream = IterableQueue() - self.stream_msg_resp = self.client_stub.StreamChannel( - iter(self.requests_stream)) - self.proto_dump_file = proto_dump_file - self.digest_thread = Thread(target=self.receive_digests) - - def start_digest_listeners(self): - logger.info( - 'Starting mac_learn_digest Digest for device [%s] named [%s]', - self.grpc_addr, self.name) - digest_entry, digest_info = self.p4info_helper.build_digest_entry( - digest_name="mac_learn_digest") - self.write_digest_entry(digest_entry) - - logger.info('Starting digest threads') - self.digest_thread.start() - - def stop_digest_listeners(self): - if 'arch' in self.sw_info and self.sw_info['arch'] == 'tofino': - logger.info('Tofino currently not supporting digests') - pass - else: - self.requests_stream.close() - self.stream_msg_resp.cancel() - self.digest_thread.join() - - def receive_digests(self): - """ - Runnable method for self.digest_thread - """ - logger.info("Started listening digest thread on device [%s] with " - "name [%s]", self.grpc_addr, self.name) - while True: - try: - logger.debug('Requesting digests from device [%s]', - self.grpc_addr) - digests = self.digest_list() - logger.debug('digests from device [%s] - [%s]', - self.grpc_addr, digests) - digest_data = digests.digest.data - logger.debug('Received digest data from device [%s]: [%s]', - self.grpc_addr, digest_data) - self.interpret_digest(digest_data) - logger.debug('Interpreted digest data') - except Exception as e: - logger.error( - 'Unexpected error reading digest from device [%s] - [%s]', - self.grpc_addr, e) - - def interpret_digest(self, digest_data): - logger.debug("Digest data from switch [%s] - [%s]", - self.name, digest_data) - - if not digest_data or len(digest_data) == 0: - logger.warning('No digest data to process') - return - for members in digest_data: - logger.debug("Digest members: %s", members) - if members.WhichOneof('data') == 'struct': - source_mac = decode_mac(members.struct.members[0].bitstring) - if source_mac: - logger.debug('Digest MAC Address is: %s', source_mac) - ingress_port = int( - codecs.encode(members.struct.members[1].bitstring, - 'hex'), 16) - logger.debug('Digest Ingress Port is %s', ingress_port) - self.add_data_forward(source_mac, ingress_port) - else: - logger.warning('Could not retrieve source_mac from digest') - else: - logger.warning('Digest could not be processed - [%s]', - digest_data) - - logger.info('Completed digest processing') - - def get_table_entry(self, table_name, action_name, match_fields, - action_params, ingress_class=True): - logger.info("Access P4 table [%s]", table_name) - if ingress_class: - tbl_class = self.p4_ingress - else: - tbl_class = self.p4_egress - - return self.p4info_helper.build_table_entry( - table_name='{}.{}'.format(tbl_class, table_name), - match_fields=match_fields, - action_name='{}.{}'.format(tbl_class, action_name), - action_params=action_params) - - def insert_p4_table_entry(self, table_name, action_name, match_fields, - action_params, ingress_class=True, - election_high=0, election_low=1): - - table_entry = self.get_table_entry( - table_name, action_name, match_fields, action_params, - ingress_class) - logger.debug( - 'Writing table entry to device [%s] table [%s], ' - 'with action name - [%s], ' - 'match fields - [%s], action_params - [%s]', - self.grpc_addr, table_name, action_name, match_fields, - action_params) - self.write_table_entry(table_entry=table_entry, - election_high=election_high, - election_low=election_low) - - def delete_p4_table_entry(self, table_name, action_name, match_fields, - action_params=None, ingress_class=True, - election_high=0, election_low=1): - - table_entry = self.get_table_entry( - table_name, action_name, match_fields, action_params, - ingress_class) - logger.debug( - 'Deleting table entry to device [%s] table [%s], ' - 'with action name - [%s], ' - 'match fields - [%s], action_params - [%s]', - self.grpc_addr, table_name, action_name, match_fields, - action_params) - self.delete_table_entry(table_entry, election_high, election_low) - - def __get_data_forward_table_entry(self, dst_mac, egress_port): - logger.info( - 'Adding data forward to device [%s] with destination MAC ' - '- [%s] and egress port - [%s]', - self.grpc_addr, dst_mac, egress_port) - - if self.sw_info['type'] == GATEWAY_CTRL_KEY: - action_params = { - 'port': egress_port, - 'switch_mac': self.mac - } - else: - action_params = {'port': egress_port} - - table_name = '{}.data_forward_t'.format(self.p4_ingress) - - table_keys = self.get_data_forward_macs() - logger.debug('Table keys to [%s] - [%s] on device [%s]', - table_name, table_keys, self.grpc_addr) - - if dst_mac not in table_keys: - logger.info('Returning table entry for data_forward_t insertion') - return self.p4info_helper.build_table_entry( - table_name=table_name, - match_fields={ - 'hdr.ethernet.dst_mac': dst_mac - }, - action_name='{}.data_forward'.format(self.p4_ingress), - action_params=action_params - ) - elif not egress_port: - logger.info('Returning table entry for data_forward_t deletion') - return self.p4info_helper.build_table_entry( - table_name=table_name, - match_fields={ - 'hdr.ethernet.dst_mac': dst_mac - }, - action_name='{}.data_forward'.format(self.p4_ingress), - ) - else: - logger.info( - 'Data forward entry already inserted with key - [%s] ' - 'on device [%s]', - dst_mac, self.grpc_addr) - return None - - def add_data_forward(self, dst_mac, egress_port): - table_entry = self.__get_data_forward_table_entry(dst_mac, egress_port) - if table_entry: - self.write_table_entry(table_entry) - return True - else: - logger.warning( - 'Unable to insert data_forward request with mac - [%s] & ' - 'port - [%s]', dst_mac, egress_port) - return False - - def del_data_forward(self, dst_mac): - table_entry = self.__get_data_forward_table_entry(dst_mac, None) - if table_entry: - self.delete_table_entry(table_entry) - return True - else: - logger.warning( - 'Unable to delete data_forward request with mac - [%s]', - dst_mac) - return False - - def add_data_inspection(self, dev_id, dev_mac): - logger.info( - 'Adding data inspection to device [%s] with ID ' - '- [%s] and mac - [%s]', - self.device_id, self.int_device_id, dev_mac) - action_params = { - 'device': self.int_device_id, - 'switch_id': self.sw_info['id'] - } - table_entry = self.p4info_helper.build_table_entry( - table_name='{}.data_inspection_t'.format(self.p4_ingress), - match_fields={ - 'hdr.ethernet.src_mac': dev_mac, - }, - action_name='{}.data_inspect_packet'.format( - self.p4_ingress), - action_params=action_params - ) - self.write_table_entry(table_entry) - - logger.info( - 'Installed Northbound Packet Inspection for device with' - ' MAC - [%s] with action params - [%s]', - dev_mac, action_params) - - def del_data_inspection(self, dev_id, dev_mac): - logger.info( - 'Removing data inspection to device [%s] with ID ' - '- [%s] and mac - [%s]', self.device_id, dev_id, dev_mac) - table_entry = self.p4info_helper.build_table_entry( - table_name='{}.data_inspection_t'.format(self.p4_ingress), - match_fields={ - 'hdr.ethernet.src_mac': dev_mac, - }, - action_name='{}.data_inspect_packet'.format( - self.p4_ingress), - ) - self.delete_table_entry(table_entry) - - logger.info( - 'Installed Northbound Packet Inspection for device with' - ' MAC - [%s]', dev_mac) - - def add_switch_id(self): - pass - - def build_device_config(self): - if self.sw_info['type'] == 'tofino': - return self.__build_device_config_tofino() - else: - return self.__build_device_config_bmv2() - - def __build_device_config_bmv2(self): - runtime_json = self.sw_info['runtime_json'] - logger.info('Building device [%s] config with file - [%s]', - self.grpc_addr, runtime_json) - - device_config = p4config_pb2.P4DeviceConfig() - device_config.reassign = True - with open(runtime_json) as f: - file_data = f.read() - device_config.device_data = bytes(file_data, 'utf-8') - return device_config - - def __build_device_config_tofino(self): - """ - Builds the device config for Tofino - """ - prog_name = self.sw_info['type'] - bin_path = self.sw_info['bin_path'] - cxt_json_path = self.sw_info['cxt_json_path'] - logger.info( - 'Building device configuration for program - [%s], bin_path - [%s]' - ', and cxt_json_path - [%s]', prog_name, bin_path, cxt_json_path) - prog_name = prog_name.encode('utf-8') - device_config = p4config_pb2.P4DeviceConfig() - device_config.reassign = True - with open(bin_path, 'rb') as bin_f: - with open(cxt_json_path, 'r') as cxt_json_f: - device_config.device_data = "" - device_config.device_data += struct.pack("' + logfile + ' 2>&1 & echo $! >> ' + f.name) - pid = int(f.read()) - logger.debug("P4 switch {} PID is {}.".format(self.name, pid)) - if not self.check_switch_started(pid): - raise Exception("P4 switch {} did not start correctly.\n".format( - self.name)) - logger.info("P4 switch %s has been started.", self.name)