Repository navigation
Expand file tree
/
Copy pathcli.py
More file actions
executable file
·115 lines (101 loc) · 3.41 KB
/
Copy pathcli.py
File metadata and controls
executable file
·115 lines (101 loc) · 3.41 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
import json
import warnings
from typing import Optional
import click
# urllib3 v2 (pulled in via requests) may emit a startup warning on some Python builds
# (e.g., LibreSSL): "urllib3 v2 only supports OpenSSL 1.1.1+ ..."
# We suppress only that specific warning in the CLI to keep output clean.
warnings.filterwarnings(
"ignore",
message=r".*urllib3 v2 only supports OpenSSL 1\.1\.1\+.*",
)
from glassgen import generate # noqa
@click.group()
def cli():
"""GlassGen - Flexible synthetic data generation service"""
pass
@cli.command()
@click.option(
"--config",
"-c",
type=click.Path(exists=True),
required=True,
help="Path to configuration file",
)
def generate_data(config: str):
"""Generate synthetic data based on the specified configuration file"""
try:
with open(config, "r") as f:
config_data = json.load(f)
# Generate data
num_records = config_data.get("generator", {}).get("num_records", 1000)
click.echo(f"Generating {num_records} records...")
generate(config=config_data)
output_path = (
config_data.get("sink", {}).get("params", {}).get("path", "output")
)
click.echo(f"Data generation completed. Output saved to {output_path}")
except Exception as e:
click.echo(f"Error: {str(e)}", err=True)
raise click.Abort() from e
@cli.command()
@click.option(
"--type",
"-t",
type=click.Choice(["csv", "kafka"]),
default="csv",
help="Configuration type",
)
@click.option("--output", "-o", type=click.Path(), help="Output file path")
def init_config(type: str, output: Optional[str]):
"""Initialize a new configuration file"""
config_templates = {
"csv": {
"schema": {
"name": "$name",
"email": "$email",
"country": "$country",
"id": "$uuid",
"address": "$address",
"phone": "$phone_number",
"job": "$job",
"company": "$company",
},
"sink": {"type": "csv", "params": {"path": "output.csv"}},
"generator": {"rps": 500, "num_records": 1000},
},
"kafka": {
"schema": {
"name": "$name",
"email": "$email",
"country": "$country",
"id": "$uuid",
"address": "$address",
"phone": "$phone_number",
"job": "$job",
"company": "$company",
},
"sink": {
"type": "kafka",
"params": {
"bootstrap.servers": "your-kafka-host:9092",
"topic": "glassgen-data",
"security.protocol": "SASL_SSL",
"sasl.mechanism": "PLAIN",
"sasl.username": "your-api-key",
"sasl.password": "your-api-secret",
},
},
"generator": {"rps": 500, "num_records": 1000},
},
}
output_path = output or f"config.{type}.json"
try:
with open(output_path, "w") as f:
json.dump(config_templates[type], f, indent=4)
click.echo(f"Configuration file created at {output_path}")
except Exception as e:
click.echo(f"Error creating configuration file: {str(e)}", err=True)
raise click.Abort() from e
if __name__ == "__main__":
cli()