diff --git a/README.md b/README.md index 983b854..5c156d9 100644 --- a/README.md +++ b/README.md @@ -196,6 +196,81 @@ $ diff-jsonl data-on-master.jsonl data-on-branch.jsonl *************** ``` +singer-analyze-record-fields +---------- +To get an idea of how many total fields a stream of records has, you can use `singer-analyze-record-fields` to count the unique paths. This tool will count the total number of unique paths observed in a set of records, and recusively count the number of unique nested paths with a depth = 3 + +Currently this only works for a single stream, but it can eventually be +extended to support multiple streams + +To use `singer-analyze-record-fields`, first run the tap and redirect the output to a file: +``` +$ --config --properties --catalog --state > records.txt +``` + +Then run `singer-analyze-record-fields` with the file as input +``` +$ singer-analyze-record-fields < records.txt +``` +#### A good run: +``` +$ singer-analyze-record-fields < records.txt +===================================================== +Total Fields: 1575 + +Level 1 ++------------------+-------------------+ +| Path | Num Nested Fields | ++------------------+-------------------+ +| data | 1566 | +| created | 1 | +| api_version | 1 | +| type | 1 | +| livemode | 1 | +| id | 1 | +| updated | 1 | +| object | 1 | +| pending_webhooks | 1 | +| request | 1 | ++------------------+-------------------+ + +Level 2 ++--------------------------+-------------------+ +| Path | Num Nested Fields | ++--------------------------+-------------------+ +| data/object | 1332 | +| data/previous_attributes | 233 | ++--------------------------+-------------------+ + +Level 3 ++------------------------------------+-------------------+ +| Path | Num Nested Fields | ++------------------------------------+-------------------+ +| data/object/metadata | 730 | +| data/object/subscriptions | 89 | +| data/previous_attributes/metadata | 79 | +| data/previous_attributes/lines | 49 | +| data/object/lines | 49 | +| data/object/card | 32 | +| data/object/items | 31 | +| data/previous_attributes/items | 31 | +| data/object/cards | 24 | +| data/object/sources | 24 | +| data/object/source | 24 | +| data/previous_attributes/plan | 23 | +| data/object/plan | 23 | +| data/object/balance_transactions | 21 | +| data/object/payment_method_details | 16 | +| data/object/refunds | 14 | +| data/object/bank_account | 13 | +| data/object/billing_details | 11 | +| data/object/outcome | 11 | +| data/object/support_address | 7 | +| data/object/ip_address_location | 6 | ++------------------------------------+-------------------+ +===================================================== +``` + License ------- diff --git a/setup.py b/setup.py index 717c156..d60da99 100644 --- a/setup.py +++ b/setup.py @@ -22,6 +22,7 @@ singer-check-tap=singertools.check_tap:main singer-release=singertools.release:main diff-jsonl=singertools.diff_jsonl:main + singer-analyze-record-fields=singertools.analyze_record_fields:main ''', include_package_data=True, ) diff --git a/singertools/analyze_record_fields.py b/singertools/analyze_record_fields.py new file mode 100644 index 0000000..18c7c34 --- /dev/null +++ b/singertools/analyze_record_fields.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 + +from collections import Counter +import json +import sys +from terminaltables import AsciiTable + + +NUM_LEVELS = 3 +NUM_TO_PRINT_PER_LEVEL = 20 + +def get_paths_from_rec(rec, path_to_rec): + unique_fields = set() + for this_key in rec.keys(): + this_path = path_to_rec + "/" + this_key + unique_fields.add(this_path) + if isinstance(rec[this_key],dict): + unique_fields = unique_fields.union(get_paths_from_rec(rec[this_key], this_path)) + elif isinstance(rec[this_key], list): + for list_entry in rec[this_key]: + if isinstance(list_entry, dict): + unique_fields = unique_fields.union(get_paths_from_rec(list_entry, this_path)) + return unique_fields + +def get_counts(unique_fields, level): + split_fields = [f.split('/') for f in unique_fields] + nested_fields = [f for f in split_fields if len(f) > level] + nested_fields_level = ['/'.join(f[1:(level+1)]) for f in nested_fields] + return dict(Counter(nested_fields_level)) + +def print_summary(unique_fields): + count_level_map = {} + print('=====================================================') + for level in range(0,(NUM_LEVELS+1)): + + if level != 0: + print('\nLevel ' + str(level)) + counts = get_counts(unique_fields, level) + sorted_counts = sorted(counts.items(), key=lambda x: x[1], reverse=True) + num_sorted_output = 0 + + if level == 0: + print("Total Fields: " + str(sorted_counts[0][1])) + else: + headers = [['Path', 'Num Nested Fields']] + rows = [[f[0], f[1]] for f in sorted_counts[0:(NUM_TO_PRINT_PER_LEVEL+1)]] + data = headers + rows + table = AsciiTable(data) + print(table.table) + + print('=====================================================') + +def main(): + num_recs = 0 + unique_fields = set() + for line in sys.stdin: + rec = json.loads(line) + if rec['type'] == 'RECORD': + rec_unique_fields = get_paths_from_rec(rec['record'], "") + unique_fields = unique_fields.union(rec_unique_fields) + num_recs += 1 + print_summary(unique_fields)