Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion src/backend/src/common/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,5 +47,4 @@
'ErrorHandlingMiddleware',
'CachingWorkspaceClient',
'get_workspace_client',
'get_sql_connection'
]
103 changes: 0 additions & 103 deletions src/backend/src/controller/data_contracts_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -263,109 +263,6 @@ def validate_odcs_format(self, data: Dict) -> bool:
# Add more validation as needed
return True

def create_from_odcs(self, data: Dict) -> DataContract:
"""Create a data contract from ODCS v3 format"""
# Convert ODCS metadata
metadata = Metadata(
domain=data.get('domain', 'default'),
owner=data.get('owner', 'Unknown'),
tags=data.get('tags', {}),
business_description=data.get('description', '')
)

# Convert ODCS datasets
datasets = []
for ds_data in data.get('datasets', []):
# Convert schema
columns = []
for col in ds_data.get('schema', {}).get('columns', []):
columns.append(ColumnDefinition(
name=col['name'],
data_type=self._map_odcs_type(col['type']),
comment=col.get('description', ''),
nullable=col.get('nullable', True),
is_unique=col.get('unique', False)
))

schema = DatasetSchema(
columns=columns,
primary_key=ds_data.get('schema', {}).get('primaryKey', []),
version=ds_data.get('version', '1.0')
)

# Convert quality rules
quality = Quality(
rules=ds_data.get('quality', {}).get('rules', []),
scores=ds_data.get('quality', {}).get('scores', {}),
metrics=ds_data.get('quality', {}).get('metrics', {})
)

# Convert security
security = Security(
classification=self._map_odcs_classification(
ds_data.get('security', {}).get('classification', 'INTERNAL')
),
pii_data=ds_data.get('security', {}).get('containsPII', False),
compliance_labels=ds_data.get('security', {}).get('complianceLabels', [])
)

# Create dataset
datasets.append(Dataset(
name=ds_data['name'],
type=ds_data.get('type', 'table'),
schema=schema,
metadata=metadata,
quality=quality,
security=security,
lifecycle=DatasetLifecycle(),
description=ds_data.get('description', '')
))

# Create and return contract
return self.create_contract(
name=data['name'],
contract_text=json.dumps(data),
format='json',
version=data['version'],
metadata=metadata,
datasets=datasets,
validation_rules=data.get('validationRules', []),
effective_from=self._parse_odcs_date(data.get('effectiveFrom')),
effective_until=self._parse_odcs_date(data.get('effectiveUntil')),
terms_and_conditions=data.get('termsAndConditions', '')
)

def _map_odcs_type(self, odcs_type: str) -> DataType:
"""Map ODCS data types to internal types"""
type_mapping = {
'string': DataType.STRING,
'integer': DataType.INTEGER,
'number': DataType.DOUBLE,
'boolean': DataType.BOOLEAN,
'date': DataType.DATE,
'timestamp': DataType.TIMESTAMP
}
return type_mapping.get(odcs_type.lower(), DataType.STRING)

def _map_odcs_classification(self, classification: str) -> SecurityClassification:
"""Map ODCS security classifications"""
class_mapping = {
'public': SecurityClassification.PUBLIC,
'internal': SecurityClassification.INTERNAL,
'confidential': SecurityClassification.CONFIDENTIAL,
'restricted': SecurityClassification.RESTRICTED
}
return class_mapping.get(classification.lower(), SecurityClassification.INTERNAL)

def _parse_odcs_date(self, date_str: Optional[str]) -> Optional[datetime]:
"""Parse ODCS date format"""
if not date_str:
return None
try:
return datetime.fromisoformat(date_str.replace('Z', '+00:00'))
except ValueError:
return None

def to_odcs_format(self, contract: DataContract) -> Dict:
"""Convert a data contract to ODCS v3 format"""

Expand Down
8 changes: 2 additions & 6 deletions src/backend/src/models/data_domains.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,7 @@ class DataDomainBase(BaseModel):

@field_validator('tags', mode='before')
def validate_tags(cls, v):
if v is None:
return v
# All tags should now be AssignedTagCreate objects
# All tags should now be AssignedTagCreate objects; passed through as-is.
return v

# --- Create Model --- #
Expand All @@ -43,9 +41,7 @@ class DataDomainUpdate(BaseModel):

@field_validator('tags', mode='before')
def validate_update_tags(cls, v):
if v is None:
return v
# All tags should now be AssignedTagCreate objects
# All tags should now be AssignedTagCreate objects; passed through as-is.
return v

# --- Read Model (includes DB fields) --- #
Expand Down
4 changes: 2 additions & 2 deletions src/backend/src/utils/startup_tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from sqlalchemy.orm import Session
from sqlalchemy import text

from src.common.config import get_settings, Settings
from src.common.config import get_settings
from src.common.logging import get_logger
from src.common.database import init_db, get_session_factory, Base, engine, cleanup_db
from src.common.workspace_client import get_workspace_client
Expand Down Expand Up @@ -117,7 +117,7 @@ def load_demo_data_from_sql() -> bool:
return False


def initialize_database(settings: Settings): # Keep settings param for future use if needed
def initialize_database():
"""Initializes the database by calling the main init_db function."""
logger.info("Triggering database initialization...")
try:
Expand Down
Loading