Add analyze record fields task

README.md

@@ -196,6 +196,81 @@ $ diff-jsonl data-on-master.jsonl data-on-branch.jsonl
 ***************
 ```
 
+singer-analyze-record-fields
+----------
+To get an idea of how many total fields a stream of records has, you can use `singer-analyze-record-fields` to count the unique paths. This tool will count the total number of unique paths observed in a set of records, and recusively count the number of unique nested paths with a depth = 3  
+
+Currently this only works for a single stream, but it can eventually be
+extended to support multiple streams
+
+To use `singer-analyze-record-fields`, first run the tap and redirect the output to a file:
+```
+$ <tap-foo> --config <config_file> --properties <catalog_file> --catalog <catalog_file> --state <state_file> > records.txt
+```
+
+Then run `singer-analyze-record-fields` with the file as input
+```
+$ singer-analyze-record-fields < records.txt
+```
+#### A good run:
+```
+$ singer-analyze-record-fields < records.txt
+=====================================================
+Total Fields: 1575
+
+Level 1
++------------------+-------------------+
+| Path             | Num Nested Fields |
++------------------+-------------------+
+| data             | 1566              |
+| created          | 1                 |
+| api_version      | 1                 |
+| type             | 1                 |
+| livemode         | 1                 |
+| id               | 1                 |
+| updated          | 1                 |
+| object           | 1                 |
+| pending_webhooks | 1                 |
+| request          | 1                 |
++------------------+-------------------+
+
+Level 2
++--------------------------+-------------------+
+| Path                     | Num Nested Fields |
++--------------------------+-------------------+
+| data/object              | 1332              |
+| data/previous_attributes | 233               |
++--------------------------+-------------------+
+
+Level 3
++------------------------------------+-------------------+
+| Path                               | Num Nested Fields |
++------------------------------------+-------------------+
+| data/object/metadata               | 730               |
+| data/object/subscriptions          | 89                |
+| data/previous_attributes/metadata  | 79                |
+| data/previous_attributes/lines     | 49                |
+| data/object/lines                  | 49                |
+| data/object/card                   | 32                |
+| data/object/items                  | 31                |
+| data/previous_attributes/items     | 31                |
+| data/object/cards                  | 24                |
+| data/object/sources                | 24                |
+| data/object/source                 | 24                |
+| data/previous_attributes/plan      | 23                |
+| data/object/plan                   | 23                |
+| data/object/balance_transactions   | 21                |
+| data/object/payment_method_details | 16                |
+| data/object/refunds                | 14                |
+| data/object/bank_account           | 13                |
+| data/object/billing_details        | 11                |
+| data/object/outcome                | 11                |
+| data/object/support_address        | 7                 |
+| data/object/ip_address_location    | 6                 |
++------------------------------------+-------------------+
+=====================================================
+```
+
 License
 -------
 

setup.py

@@ -22,6 +22,7 @@
           singer-check-tap=singertools.check_tap:main
           singer-release=singertools.release:main
           diff-jsonl=singertools.diff_jsonl:main
+          singer-analyze-record-fields=singertools.analyze_record_fields:main
       ''',
       include_package_data=True,
 )

singertools/analyze_record_fields.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+
+from collections import Counter
+import json
+import sys
+from terminaltables import AsciiTable
+
+
+NUM_LEVELS = 3
+NUM_TO_PRINT_PER_LEVEL = 20
+
+def get_paths_from_rec(rec, path_to_rec):
+    unique_fields = set()
+    for this_key in rec.keys():
+        this_path = path_to_rec + "/" + this_key
+        unique_fields.add(this_path)
+        if isinstance(rec[this_key],dict):
+            unique_fields = unique_fields.union(get_paths_from_rec(rec[this_key], this_path))
+        elif isinstance(rec[this_key], list):
+            for list_entry in rec[this_key]:
+                if isinstance(list_entry, dict):
+                    unique_fields = unique_fields.union(get_paths_from_rec(list_entry, this_path))
+    return unique_fields
+
+def get_counts(unique_fields, level):
+    split_fields = [f.split('/') for f in unique_fields]
+    nested_fields = [f for f in split_fields if len(f) > level]
+    nested_fields_level = ['/'.join(f[1:(level+1)]) for f in nested_fields]
+    return dict(Counter(nested_fields_level))
+
+def print_summary(unique_fields):
+    count_level_map = {}
+    print('=====================================================')
+    for level in range(0,(NUM_LEVELS+1)):
+
+        if level != 0:
+            print('\nLevel ' + str(level))
+        counts = get_counts(unique_fields, level)
+        sorted_counts = sorted(counts.items(), key=lambda x: x[1], reverse=True)
+        num_sorted_output = 0
+
+        if level == 0:
+            print("Total Fields: " + str(sorted_counts[0][1]))            
+        else:
+            headers = [['Path', 'Num Nested Fields']]
+            rows = [[f[0], f[1]] for f in sorted_counts[0:(NUM_TO_PRINT_PER_LEVEL+1)]]
+            data = headers + rows
+            table = AsciiTable(data)
+            print(table.table)
+
+    print('=====================================================')
+
+def main():
+    num_recs = 0
+    unique_fields = set()
+    for line in sys.stdin:
+       rec = json.loads(line)
+       if rec['type'] == 'RECORD':
+           rec_unique_fields = get_paths_from_rec(rec['record'], "")
+           unique_fields = unique_fields.union(rec_unique_fields)
+           num_recs += 1
+    print_summary(unique_fields)