diff --git a/specifyweb/backend/businessrules/migration_utils.py b/specifyweb/backend/businessrules/migration_utils.py index c490dcfbe41..0bea9455998 100644 --- a/specifyweb/backend/businessrules/migration_utils.py +++ b/specifyweb/backend/businessrules/migration_utils.py @@ -1,71 +1,31 @@ -from typing import Tuple, List - -from specifyweb.backend.businessrules.uniqueness_rules import create_uniqueness_rule - - def catnum_rule_editable(apps, schema_editor=None): """ Find any CollectionObject catalogNumber must be unique to Collection rules which are readonly on the frontend (have isDatabaseConstraint=True) and set their isDatabaseConstraint=False. - + Generally should be run only after migration businessrules/0003 has been applied """ UniquenessRule = apps.get_model("businessrules", "UniquenessRule") - - model_rules = UniquenessRule.objects.filter(modelName="Collectionobject", isDatabaseConstraint=True) - catalog_number_rules: List[int] = [] - for rule in model_rules: + model_rules = UniquenessRule.objects.filter( + modelName="Collectionobject", + isDatabaseConstraint=True + ) + + catalog_number_rules: list[int] = [] + for rule in model_rules: rule_fields = rule.uniquenessrulefield_set.all() fields = rule_fields.filter(isScope=False) scopes = rule_fields.filter(isScope=True) - # We're only interested in the rule "CollectionObject catalogNumber + # We're only interested in the rule "CollectionObject catalogNumber # must be unique to Collection" - # We check for length of fields and scopes because get() raises an + # We check for length of fields and scopes because get() raises an # exception if more than one result is returned if (len(fields) == 1 and len(scopes) == 1) and (fields.get().fieldPath.lower() == "catalognumber" and scopes.get().fieldPath.lower() == "collection"): catalog_number_rules.append(rule.id) - + rules_to_update = UniquenessRule.objects.filter(id__in=catalog_number_rules) rules_to_update.update(isDatabaseConstraint=False) - - -def catnum_rule_uneditable(apps, schema_editor=None): - """ Find any CollectionObject catalogNumber must be unique to Collection - rules which are editable on the frontend (have isDatabaseConstraint=False) - and set their isDatabaseConstraint=True. - - Generally should be run when migration businessrules/0003 is being reverted - """ - Discipline = apps.get_model("specify", "Discipline") - UniquenessRule = apps.get_model("businessrules", "UniquenessRule") - - for discipline in Discipline.objects.all(): - model_rules = UniquenessRule.objects.filter(modelName="Collectionobject", discipline_id=discipline.id, isDatabaseConstraint=False) - - has_catalognumber_rule = False - for rule in model_rules: - rule_fields = rule.uniquenessrulefield_set.all() - - fields = rule_fields.filter(isScope=False) - scopes = rule_fields.filter(isScope=True) - - # We're only interested in the rule "CollectionObject catalogNumber - # must be unique to Collection" - # We check for length of fields and scopes because get() raises an - # exception if more than one result is returned - if (len(fields) == 1 and len(scopes) == 1) and (fields.get().fieldPath.lower() == "catalognumber" and scopes.get().fieldPath.lower() == "collection"): - has_catalognumber_rule = True - - if not has_catalognumber_rule: - create_uniqueness_rule( - model_name="Collectionobject", - discipline=discipline, - is_database_constraint=True, - fields=["catalogNumber"], - scopes=["collection"], - registry=apps, - ) diff --git a/specifyweb/backend/businessrules/migrations/0004_catnum_uniquerule.py b/specifyweb/backend/businessrules/migrations/0004_catnum_uniquerule.py index 77f61d74103..b35a3283d67 100644 --- a/specifyweb/backend/businessrules/migrations/0004_catnum_uniquerule.py +++ b/specifyweb/backend/businessrules/migrations/0004_catnum_uniquerule.py @@ -1,42 +1,57 @@ from django.db import migrations -from specifyweb.backend.businessrules.migration_utils import catnum_rule_editable, catnum_rule_uneditable +from specifyweb.backend.businessrules.migration_utils import catnum_rule_editable from specifyweb.backend.businessrules.uniqueness_rules import create_uniqueness_rule -def catnum_rule_editable(apps, schema_editor): - UniquenessRule = apps.get_model('businessrules', 'UniquenessRule') - UniquenessRuleField = apps.get_model('businessrules', 'UniquenessRuleField') - - candidate_rules_with_field: tuple[int] = tuple(UniquenessRuleField.objects.filter(uniquenessrule__modelName__iexact='collectionobject', uniquenessrule__isDatabaseConstraint=True, fieldPath__iexact='catalognumber', isScope=False).values_list('uniquenessrule_id', flat=True)) +def catnum_rule_uneditable(apps, schema_editor): + """ Find any CollectionObject catalogNumber must be unique to Collection + rules which are editable on the frontend (have isDatabaseConstraint=False) + and set their isDatabaseConstraint=True. - candidate_rules_with_scope: tuple[int] = tuple(UniquenessRuleField.objects.filter(uniquenessrule_id__in=candidate_rules_with_field, fieldPath__iexact='collection', isScope=True).values_list('uniquenessrule_id', flat=True)) + Generally should be run when migration businessrules/0003 is being reverted + """ + Discipline = apps.get_model("specify", "Discipline") + UniquenessRule = apps.get_model("businessrules", "UniquenessRule") - candidate_rules = UniquenessRule.objects.filter(id__in=candidate_rules_with_scope) - candidate_rules.update(isDatabaseConstraint=False) + for discipline in Discipline.objects.all(): + # REFACTOR: Some of these queries should be able to be combined to + # improve performance and limit how often we need to hit the database + model_rules = UniquenessRule.objects.filter( + modelName="Collectionobject", + discipline_id=discipline.id, + isDatabaseConstraint=False + ) -def catnum_rule_uneditable(apps, schema_editor): - Discipline = apps.get_model('specify', 'Discipline') - UniquenessRule = apps.get_model('businessrules', 'UniquenessRule') - UniquenessRuleField = apps.get_model('businessrules', 'UniquenessRuleField') + has_catalognumber_rule = False + matching_rule_ids: list[int] = [] + for rule in model_rules: + rule_fields = rule.uniquenessrulefield_set.all() - for discipline in Discipline.objects.all(): - candidate_rules_with_field: tuple[int] = tuple(UniquenessRuleField.objects.filter(uniquenessrule__modelName__iexact='collectionobject', uniquenessrule__discipline=discipline.id, uniquenessrule__isDatabaseConstraint=False, fieldPath__iexact='catalognumber', isScope=False).values_list('uniquenessrule_id', flat=True)) + fields = rule_fields.filter(isScope=False) + scopes = rule_fields.filter(isScope=True) - candidate_rules_with_scope: tuple[int] = tuple(UniquenessRuleField.objects.filter(uniquenessrule_id__in=candidate_rules_with_field, fieldPath__iexact='collection', isScope=True).values_list('uniquenessrule_id', flat=True)) + # We're only interested in the rule "CollectionObject catalogNumber + # must be unique to Collection" + # We check for length of fields and scopes because get() raises an + # exception if more than one result is returned + if (len(fields) == 1 and len(scopes) == 1) and (fields.get().fieldPath.lower() == "catalognumber" and scopes.get().fieldPath.lower() == "collection"): + has_catalognumber_rule = True + matching_rule_ids.append(rule.id) - candidate_rules = UniquenessRule.objects.filter(id__in=candidate_rules_with_scope) - if len(candidate_rules) == 0: + if has_catalognumber_rule: + UniquenessRule.objects.filter( + id__in=matching_rule_ids).update(isDatabaseConstraint=True) + else: create_uniqueness_rule( - model_name='Collectionobject', + model_name="Collectionobject", discipline=discipline, is_database_constraint=True, - fields=['catalogNumber'], - scopes=['collection'], - registry=apps + fields=["catalogNumber"], + scopes=["collection"], + registry=apps, ) - else: - candidate_rules.update(isDatabaseConstraint=True) + class Migration(migrations.Migration): dependencies = [ @@ -44,5 +59,6 @@ class Migration(migrations.Migration): ] operations = [ - migrations.RunPython(catnum_rule_editable, catnum_rule_uneditable, atomic=True) + migrations.RunPython(catnum_rule_editable, + catnum_rule_uneditable, atomic=True) ] diff --git a/specifyweb/backend/businessrules/uniqueness_rules.py b/specifyweb/backend/businessrules/uniqueness_rules.py index 34ec4af7857..d85b1c38e46 100644 --- a/specifyweb/backend/businessrules/uniqueness_rules.py +++ b/specifyweb/backend/businessrules/uniqueness_rules.py @@ -99,7 +99,11 @@ def _initial_businessrules_migration_applied(): ).applied_migrations() ) +def businessrule_app_is_ready(registry): + return any(app.label == 'businessrules' for app in registry.get_app_configs()) +# BUG: If we reverse past the initial businessrule migration, Specify can still +# consider the migration applied within those earlier migrations def _cached_businessrules_migration_applied() -> bool: cache_key = "default" cache_is_active, is_set = _uniqueness_migration_cache.get(cache_key, default=False) @@ -188,12 +192,19 @@ def validate_unique(model, instance): f"Skipping uniqueness rule check on non-Specify model: '{model_name}'") return - if not _cached_businessrules_migration_applied(): - return - # We can't directly use the main app registry in the context of migrations, which uses fake models registry = model._meta.apps + # If we're in a migration where businessrules have not been loaded and/or + # the initial businessrule migration has not been applied, then skip + # checking the rule for now. + # Note that the former can exist where the latter does: if we're reversing + # a migration which does not have a dependency on businessrules (so the + # businessrules app does not need to be loaded) but the businessrule + # migration is still applied + if not businessrule_app_is_ready(registry) or not _cached_businessrules_migration_applied(): + return + # REFACTOR(perf): We should look into batching UniquenessRule queries. # That is, instead of making a query to the DB for each rule, aggregate # the rules and make a "single" query. @@ -490,58 +501,48 @@ def rule_is_global(scopes: Iterable[str]) -> bool: def fix_global_default_rules(registry=None): + """ + Removes UniquenessRules that are scoped to Discipline that already exist + globally. + + There were historically cases where UniquenessRules were incorrectly + created in two places: globally and scoped to a particular discipline. + + See https://github.com/specify/specify7/pull/6308#issuecomment-3247556491 + """ UniquenessRule = registry.get_model('businessrules', 'UniquenessRule') \ if registry \ else models.UniquenessRule - UniquenessRuleField = registry.get_model('businessrules', 'UniquenessRuleField') \ - if registry \ - else models.UniquenessRuleField - - global_rule_fields = UniquenessRuleField.objects.filter( - uniquenessrule__discipline__isnull=True - ).values( - "uniquenessrule__modelName", - "uniquenessrule__isDatabaseConstraint", - "fieldPath", - "isScope", - ) - - global_rule_exists = UniquenessRule.objects.filter( - discipline__isnull=True, - modelName=OuterRef("modelName"), - isDatabaseConstraint=OuterRef("isDatabaseConstraint"), - ) - - discipline_ids = ( - UniquenessRule.objects.exclude(discipline__isnull=True) - .values_list("discipline_id", flat=True) - .distinct() - ) - for discipline_id in discipline_ids: - with transaction.atomic(): - # Delete matching fields for this discipline - matching_fields_qs = UniquenessRuleField.objects.filter( - uniquenessrule__discipline_id=discipline_id - ).filter( - Exists( - global_rule_fields.filter( - **{ - "uniquenessrule__modelName": OuterRef("uniquenessrule__modelName"), - "uniquenessrule__isDatabaseConstraint": OuterRef("uniquenessrule__isDatabaseConstraint"), - "fieldPath": OuterRef("fieldPath"), - "isScope": OuterRef("isScope"), - } - ) - ) + global_rule_signatures = { + ( + rule.modelName, + rule.isDatabaseConstraint, + frozenset( + (field.fieldPath, field.isScope) + for field in rule.uniquenessrulefield_set.all() + ), ) - matching_fields_qs.delete() - - # Delete UniquenessRule rows for this discipline that are now empty - empty_rules_qs = ( - UniquenessRule.objects.filter(discipline_id=discipline_id) - .annotate(field_count=Count("uniquenessrulefield")) - .filter(field_count=0) # now empty after field deletions - .filter(Exists(global_rule_exists)) - ) - empty_rules_qs.delete() + for rule in UniquenessRule.objects.filter( + discipline__isnull=True + ).prefetch_related("uniquenessrulefield_set") + } + + with transaction.atomic(): + # REFACTOR: See if we can simplify this even further. We should be able + # to collapse this query -> iteration -> check workflow to a single + # query. + # That would eliminate the N + 1 problem with this current approach, + # where every scoped rule needs to be evaluated. + for rule in UniquenessRule.objects.exclude(discipline__isnull=True).prefetch_related("uniquenessrulefield_set"): + signature = ( + rule.modelName, + rule.isDatabaseConstraint, + frozenset( + (field.fieldPath, field.isScope) + for field in rule.uniquenessrulefield_set.all() + ), + ) + if signature in global_rule_signatures: + rule.uniquenessrulefield_set.all().delete() + rule.delete() diff --git a/specifyweb/backend/permissions/initialize.py b/specifyweb/backend/permissions/initialize.py index a9cf2c62d5b..85fdf8f210d 100644 --- a/specifyweb/backend/permissions/initialize.py +++ b/specifyweb/backend/permissions/initialize.py @@ -53,13 +53,28 @@ def create_admins(apps=apps) -> None: UserPolicy = apps.get_model('permissions', 'UserPolicy') Specifyuser = apps.get_model('specify', 'Specifyuser') - if UserPolicy.objects.filter(collection__isnull=True, resource='%', action='%').exists(): - # don't do anything if there is already any admin. - return - users = Specifyuser.objects.all() for user in users: - if is_sp6_user_permissions_migrated(user, apps): + # REFACTOR: Try and fold the following checks into a single query to + # avoid making multiple queries per user. + # Ideally, we only make a single query to fetch all users that: + # - Are not already Institution Admins + # - Have not already seen activity in Sp 7 (don't have Sp7 permissions) + # - (The Institution Admin permission could have been intentionally + # removed) + # - Are admins in Sp 6 + + # The ordering here for checks here is intentional: it's more likely a + # user has Sp 7 permissions than being an admin, so we do the former + # check first + if is_sp6_user_permissions_migrated(user=user, apps=apps): + continue + if UserPolicy.objects.filter( + collection__isnull=True, + specifyuser_id=user.id, + resource="%", + action="%", + ).exists(): continue if is_legacy_admin(user): UserPolicy.objects.get_or_create( @@ -92,14 +107,6 @@ def assign_users_to_roles(apps=apps) -> None: results = [] with connection.cursor() as cursor: - cursor.execute(""" - SELECT COUNT(*) - FROM information_schema.tables - WHERE table_name IN ('specifyuser_spprincipal', 'spuserrole') - AND table_schema = DATABASE(); - """) - if cursor.fetchone()[0] < 2: - return # Newly created sp7 databases don't have these sp6 specific tables. cursor.execute(""" SELECT u.SpecifyUserID as user_id, @@ -112,37 +119,34 @@ def assign_users_to_roles(apps=apps) -> None: JOIN spprincipal p ON p.SpPrincipalID = up.SpPrincipalID JOIN collection c ON c.UserGroupScopeId = p.userGroupScopeID WHERE p.groupType IS NULL - AND u.SpecifyUserID NOT IN ( - SELECT ur.specifyuser_id + AND NOT EXISTS ( + SELECT 1 FROM spuserrole ur JOIN sprole r ON r.id = ur.role_id - WHERE r.collection_id = p.usergroupscopeid - ) - AND c.UserGroupScopeId NOT IN ( - SELECT DISTINCT r.collection_id - FROM spuserrole ur - JOIN sprole r ON r.id = ur.role_id - JOIN collection c ON c.UserGroupScopeId = r.collection_id + WHERE r.collection_id = c.UserGroupScopeId + AND ur.specifyuser_id = u.SpecifyUserID ); """) results = cursor.fetchall() for user_id, user_name, user_type, collection_id, collection_name in results: - if user_type not in {'Manager', 'FullAccess', 'LimitedAccess', 'Guest'}: + # REFACTOR: If we want to exlcude all other roles, why don't we write + # the exlcusion in the query rather than evaluate in Python? + if user_type not in ROLE_NAMES.keys(): continue role_name = ROLE_NAMES.get(user_type, f"{user_type} - {collection_name}") role_description = ROLE_DESCRIPTIONS.get(user_type, "No description available.") logger.info(f"Assigned user {user_name} to role {role_name} for collection {collection_name}.") - role, is_new_role = Role.objects.get_or_create( + role, _ = Role.objects.get_or_create( collection_id=collection_id, - name=role_name + name=role_name, + defaults={ + "description": role_description + } ) - if is_new_role: - role.description = role_description - role.save() UserRole.objects.get_or_create( specifyuser_id=user_id, role=role diff --git a/specifyweb/backend/stored_queries/execution.py b/specifyweb/backend/stored_queries/execution.py index 5dbb4228052..0aa0c33ea18 100644 --- a/specifyweb/backend/stored_queries/execution.py +++ b/specifyweb/backend/stored_queries/execution.py @@ -856,9 +856,8 @@ def execute( if limit: query = query.limit(limit) + log_sqlalchemy_query(query) - - log_sqlalchemy_query(query) # Debugging return {"results": apply_special_post_query_processing(query, tableid, field_specs, collection, user)} def build_query( @@ -1065,7 +1064,7 @@ def series_post_query(query, limit=40, offset=0, sort_type=0, co_id_cat_num_pair and adding a co_id colum and formatted catnum range column. Sort the results by the first catnum in the range.""" - log_sqlalchemy_query(query) # Debugging + log_sqlalchemy_query(query) def parse_catalog_for_comparing(s): def check_for_decimal(s): diff --git a/specifyweb/backend/stored_queries/utils.py b/specifyweb/backend/stored_queries/utils.py index 64cd49b1944..f25921f3023 100644 --- a/specifyweb/backend/stored_queries/utils.py +++ b/specifyweb/backend/stored_queries/utils.py @@ -7,6 +7,8 @@ from sqlalchemy.sql.selectable import Select from sqlalchemy.sql.sqltypes import NullType +from django.conf import settings + logger = logging.getLogger(__name__) def _coerce_statement(obj: Any) -> ClauseElement: @@ -50,7 +52,7 @@ def log_sqlalchemy_query( Run in the stored_queries.execute file, in the execute function, right before the return statement: from specifyweb.specify.utils import log_sqlalchemy_query; log_sqlalchemy_query(query) """ - if not logger.isEnabledFor(level): + if not logger.isEnabledFor(level) or not settings.DEBUG: return None # skip compiling and logging if we're not logging at this level dialect = dialect or mysql_dialect.dialect() diff --git a/specifyweb/settings/__init__.py b/specifyweb/settings/__init__.py index 990c5ed8ce0..5a21e5ebeed 100644 --- a/specifyweb/settings/__init__.py +++ b/specifyweb/settings/__init__.py @@ -82,6 +82,8 @@ }, } +DATABASE_ROUTERS = ["specifyweb.specify.migration_utils.router.MigrationRouter"] + DB_ALIAS = os.getenv("DJANGO_DB_ALIAS", "default") # Might want to set to "app" in the future if DB_ALIAS != "default": from copy import deepcopy diff --git a/specifyweb/specify/api/utils.py b/specifyweb/specify/api/utils.py index 1b53ff42bd7..0bd8afe81fd 100644 --- a/specifyweb/specify/api/utils.py +++ b/specifyweb/specify/api/utils.py @@ -2,6 +2,7 @@ from specifyweb.specify import models as spmodels from specifyweb.backend.businessrules.exceptions import BusinessRuleException +from django.conf import settings logger = logging.getLogger(__name__) @@ -18,17 +19,6 @@ def get_spmodel_class(model_name: str): return getattr(spmodels, attr_name) raise AttributeError(f"Model '{model_name}' not found in models module.") -def log_sqlalchemy_query(query): - # Call this function to debug the raw SQL query generated by SQLAlchemy - from sqlalchemy.dialects import mysql - compiled_query = query.statement.compile(dialect=mysql.dialect(), compile_kwargs={"literal_binds": True}) - raw_sql = str(compiled_query).replace('\n', ' ') + ';' - logger.debug('='.join(['' for _ in range(80)])) - logger.debug(raw_sql) - logger.debug('='.join(['' for _ in range(80)])) - # Run in the storred_queries.execute file, in the execute function, right before the return statement, line 546 - # from specifyweb.specify.utils import log_sqlalchemy_query; log_sqlalchemy_query(query) - def create_default_collection_types(apps, using="default"): db = using or "default" diff --git a/specifyweb/specify/management/commands/run_key_migration_functions.py b/specifyweb/specify/management/commands/run_key_migration_functions.py index 3c12ab336ed..0f61164f704 100644 --- a/specifyweb/specify/management/commands/run_key_migration_functions.py +++ b/specifyweb/specify/management/commands/run_key_migration_functions.py @@ -17,13 +17,13 @@ create_default_collection_types, create_default_discipline_for_tree_defs, fix_taxon_treedef_discipline_links, - set_discipline_for_taxon_treedefs, - fix_tectonic_unit_treedef_discipline_links + set_discipline_for_taxon_treedefs ) from specifyweb.backend.permissions.initialize import initialize from specifyweb.specify.migration_utils import update_schema_config as usc +from specifyweb.specify.migration_utils.router import use_migration_connection from specifyweb.specify.migration_utils.misc_migrations import make_selectseries_false -from specifyweb.specify.migration_utils.tectonic_ranks import create_default_tectonic_ranks, create_root_tectonic_node +from specifyweb.specify.migration_utils.tectonic_ranks import create_default_tectonic_ranks, create_root_tectonic_node, fix_tectonic_unit_treedef_discipline_links from specifyweb.backend.patches.migration_utils import apply_migrations as apply_patches logger = logging.getLogger(__name__) @@ -221,7 +221,7 @@ def add_arguments(self, parser): nargs="*", type=str, choices=tuple(self.funcs.keys()), - help=f"Optional: specify one or more functions to run", + help="Optional: specify one or more functions to run", ) parser.add_argument( "--verbose", @@ -235,7 +235,10 @@ def handle(self, *args, **options): verbose = options.get("verbose", False) try: - with transaction.atomic(): + with (transaction.atomic(), + # WARNING: With this context manager, all functions will be run + # with the Migration connection and use the Migrator user + use_migration_connection()): if len(functions) > 0: for function in functions: if function: @@ -254,6 +257,6 @@ def handle(self, *args, **options): self.stdout.write(self.style.SUCCESS(f"Applying {func_name}...")) func(self.stdout.write if verbose else None) self.stdout.write(self.style.SUCCESS(f"Applied {func_name}")) - except Exception as e: - logger.error(f"An error occurred: {e}") + except Exception: + logger.exception("An error occurred while running key migrations") raise diff --git a/specifyweb/specify/migration_utils/default_cots.py b/specifyweb/specify/migration_utils/default_cots.py index 1259f18b55e..b46098a5714 100644 --- a/specifyweb/specify/migration_utils/default_cots.py +++ b/specifyweb/specify/migration_utils/default_cots.py @@ -1,5 +1,5 @@ import logging -from django.db.models import F +from django.db.models import F, OuterRef, Subquery logger = logging.getLogger(__name__) @@ -30,45 +30,45 @@ def create_default_collection_types(apps): collection.collectionobjecttype = cot collection.save() -def create_default_discipline_for_tree_defs(apps, using='default'): +def create_default_discipline_for_tree_defs(apps): Discipline = apps.get_model('specify', 'Discipline') Institution = apps.get_model('specify', 'Institution') # Use the specified DB alias for all queries - for discipline in Discipline.objects.using(using).all(): + for discipline in Discipline.objects.all(): geography_tree_def = discipline.geographytreedef if geography_tree_def and geography_tree_def.discipline_id is None: geography_tree_def.discipline = discipline - geography_tree_def.save(using=using) + geography_tree_def.save() geologic_time_period_tree_def = discipline.geologictimeperiodtreedef if geologic_time_period_tree_def and geologic_time_period_tree_def.discipline_id is None: geologic_time_period_tree_def.discipline = discipline - geologic_time_period_tree_def.save(using=using) + geologic_time_period_tree_def.save() lithostrat_tree_def = discipline.lithostrattreedef if lithostrat_tree_def and lithostrat_tree_def.discipline_id is None: lithostrat_tree_def.discipline = discipline - lithostrat_tree_def.save(using=using) + lithostrat_tree_def.save() taxon_tree_def = discipline.taxontreedef if taxon_tree_def and taxon_tree_def.discipline_id is None: taxon_tree_def.discipline = discipline - taxon_tree_def.save(using=using) + taxon_tree_def.save() - for institution in Institution.objects.using(using).all(): + for institution in Institution.objects.all(): storage_tree_def = institution.storagetreedef if storage_tree_def and storage_tree_def.institution_id is None: storage_tree_def.institution = institution - storage_tree_def.save(using=using) + storage_tree_def.save() -def create_cogtype_type_picklist(apps, using='default'): +def create_cogtype_type_picklist(apps): Collection = apps.get_model('specify', 'Collection') Picklist = apps.get_model('specify', 'Picklist') Picklistitem = apps.get_model('specify', 'Picklistitem') - for collection in Collection.objects.using(using).all(): - cog_type_picklist, picklist_created = Picklist.objects.using(using).get_or_create( + for collection in Collection.objects.all(): + cog_type_picklist, picklist_created = Picklist.objects.get_or_create( name='SystemCOGTypes', # Default Collection Object Group Types type=0, collection=collection, @@ -79,7 +79,7 @@ def create_cogtype_type_picklist(apps, using='default'): ) if picklist_created: for cog_type in DEFAULT_COG_TYPES: - Picklistitem.objects.using(using).get_or_create( + Picklistitem.objects.get_or_create( title=cog_type, value=cog_type, picklist=cog_type_picklist @@ -108,51 +108,37 @@ def create_cotype_picklist(apps): } ) -def set_discipline_for_taxon_treedefs(apps, using='default'): +def set_discipline_for_taxon_treedefs(apps): Collectionobjecttype = apps.get_model('specify', 'Collectionobjecttype') Taxontreedef = apps.get_model('specify', 'Taxontreedef') - collection_object_types = Collectionobjecttype.objects.using(using).filter( - taxontreedef__discipline__isnull=True - ).annotate( - discipline=F('collection__discipline') + Taxontreedef.objects.filter( + discipline__isnull=True + ).update( + discipline=Subquery( + Collectionobjecttype.objects.filter( + taxontreedef=OuterRef("pk") + ).order_by("pk").values("collection__discipline")[:1] + ) ) - for cot in collection_object_types: - Taxontreedef.objects.using(using).filter(id=cot.taxontreedef_id).update(discipline=cot.discipline) - def fix_taxon_treedef_discipline_links(apps): Discipline = apps.get_model('specify', 'Discipline') Taxontreedef = apps.get_model('specify', 'Taxontreedef') - empty_taxon_treedefs = Taxontreedef.objects.filter(discipline__isnull=True) - disciplines = Discipline.objects.all() - for empty_taxon_treedef in empty_taxon_treedefs: - for discipline in disciplines: - if discipline.taxontreedef_id == empty_taxon_treedef.id: - empty_taxon_treedef.discipline = discipline - empty_taxon_treedef.save() + # If a TaxonTreeDef has a NULL DisciplineID but there's a non-NULL + # Discipline pointing to the TaxonTreeDef via Discipline -> TaxonTreeDefID, + # then set the discipline on the TaxonTreeDef to the referencing Discipline + Taxontreedef.objects.filter( + discipline__isnull=True + ).update( + discipline=Subquery( + Discipline.objects.filter( + taxontreedef=OuterRef("pk") + ).order_by("pk").values("pk")[:1] + ) + ) -def fix_tectonic_unit_treedef_discipline_links(apps): - Discipline = apps.get_model('specify', 'Discipline') - Tectonicunittreedef = apps.get_model('specify', 'Tectonicunittreedef') - - empty_tectonic_unit_treedefs = Tectonicunittreedef.objects.filter(discipline__isnull=True) - empty_disciplines = Discipline.objects.filter(tectonicunittreedef__isnull=True) - for empty_discipline in empty_disciplines: - if not empty_tectonic_unit_treedefs.exists(): - new_tectonic_unit_treedef = Tectonicunittreedef.objects.create( - name=f'{empty_discipline.name} Tectonic Unit Tree', - discipline=empty_discipline - ) - else: - empty_discipline.tectonicunittreedef = empty_tectonic_unit_treedefs.first() - empty_discipline.save() - - for empty_tectonic_unit_treedef in empty_tectonic_unit_treedefs: - if empty_disciplines.exists(): - empty_tectonic_unit_treedef.discipline = empty_disciplines.first() - empty_tectonic_unit_treedef.save() - else: - empty_tectonic_unit_treedef.discipline = empty_disciplines.last() - empty_tectonic_unit_treedef.save() + # BUG?: We're not handling the case here when Discipline has a NULL + # TaxonTreeDefID but there's a TaxonTreeDef pointing to the Discipline via + # TaxonTreeDef -> discipline diff --git a/specifyweb/specify/migration_utils/misc_migrations.py b/specifyweb/specify/migration_utils/misc_migrations.py index f7c95092346..0554a476067 100644 --- a/specifyweb/specify/migration_utils/misc_migrations.py +++ b/specifyweb/specify/migration_utils/misc_migrations.py @@ -1,5 +1,3 @@ - - def make_selectseries_false(apps): spquery = apps.get_model('specify', 'Spquery') if 'selectseries' in [field.name for field in spquery._meta.get_fields()]: diff --git a/specifyweb/specify/migration_utils/router.py b/specifyweb/specify/migration_utils/router.py new file mode 100644 index 00000000000..ae88c0723b2 --- /dev/null +++ b/specifyweb/specify/migration_utils/router.py @@ -0,0 +1,49 @@ +from contextvars import ContextVar +from contextlib import contextmanager + +_use_migration_connection = ContextVar[bool]("use_migration_connection", default=False) + +@contextmanager +def use_migration_connection(): + """ + This can be used as a decorator or context manager to tell Django to use + the 'migrations' database defined in specifyweb/settings/__init__.py + + Examples: + + ```py + @use_migration_connection() + def my_func(): + ... # For this function block, Django will use the same connection it + # uses for migrations + + with use_migration_connection(): + ... # Within this block, Django will use the same connection it uses + # for migrations + ``` + """ + token = _use_migration_connection.set(True) + try: + yield + finally: + _use_migration_connection.reset(token) + +""" +A simple MigrationRouter that automatically routes reads and writes through the +migration connection when the use_migration_connection decorator/context manager +is used. + +This is referenced by string in the DATABASE_ROUTERS Django setting within +specifyweb/settings/__init__.py + +See the Django docs on Database Routers: +https://docs.djangoproject.com/en/4.2/topics/db/multi-db/#automatic-database-routing +""" +class MigrationRouter: + def db_for_read(self, model, **hints): + if _use_migration_connection.get(): + return 'migrations' + + def db_for_write(self, model, **hints): + if _use_migration_connection.get(): + return 'migrations' diff --git a/specifyweb/specify/migration_utils/sp7_schemaconfig.py b/specifyweb/specify/migration_utils/sp7_schemaconfig.py index 38e5ba48a88..7e331639d17 100644 --- a/specifyweb/specify/migration_utils/sp7_schemaconfig.py +++ b/specifyweb/specify/migration_utils/sp7_schemaconfig.py @@ -148,7 +148,7 @@ 'RelativeAge': ['number2', 'yesno2', 'relativeAgeId', 'relativeAgePeriod', 'text1', 'agent1', 'collectionDate', 'text2', 'agent2', 'date1', 'date2', 'collectionObject', 'relativeAgeCitations', 'number1', 'yesno1'], 'CollectionObject': ['collectionObjectType', 'relativeAges', 'absoluteAges', 'cojo'], 'AbsoluteAgeCitation': ['collectionMember', 'absoluteAgeCitationId'], - 'RelativeAgeCitation': ['absoluteAgeCitationId', 'collectionMember'], + 'RelativeAgeCitation': ['relativeAgeCitationId', 'collectionMember'], 'TectonicUnit': ['collectionMember', 'nodeNumber', 'yesno1', 'tectonicUnitId', 'number1', 'yesno2', 'number2', 'rankId', 'text1'], 'TectonicUnitTreeDefItem': ['children', 'rankId', 'parent', 'treeDef', 'treeEntries', 'tectonicUnitTreeDefItemId'], 'TectonicUnitTreeDef': ['discipline', 'treeEntries', 'tectonicUnitTreeDefId'] diff --git a/specifyweb/specify/migration_utils/tectonic_ranks.py b/specifyweb/specify/migration_utils/tectonic_ranks.py index ac4c199e91c..4f23ded3ad5 100644 --- a/specifyweb/specify/migration_utils/tectonic_ranks.py +++ b/specifyweb/specify/migration_utils/tectonic_ranks.py @@ -1,157 +1,145 @@ import logging + +from django.db.models import OuterRef, Subquery, Exists + logger = logging.getLogger(__name__) +DEFAULT_RANKS = ( + {"rankid": 0, "name": "Root", "attrs": {"isenforced": True}}, + {"rankid": 10, "name": "Superstructure"}, + {"rankid": 20, "name": "Tectonic Domain"}, + {"rankid": 30, "name": "Tectonic Subdomain"}, + {"rankid": 40, "name": "Tectonic Unit"}, + {"rankid": 50, "name": "Tectonic Subunit"} +) + def create_default_tectonic_ranks(apps): TectonicUnitTreeDefItem = apps.get_model('specify', 'TectonicUnitTreeDefItem') TectonicTreeDef = apps.get_model('specify', 'TectonicUnitTreeDef') Discipline = apps.get_model('specify', 'Discipline') - disciplines = Discipline.objects.filter(tectonicunittreedef__isnull=True).exclude( - id__in=TectonicTreeDef.objects.values_list('discipline_id', flat=True) + # Create empty TectonicUnit trees for Disciplines which don't have already them + _create_tectonic_unit_for_discipline( + Discipline_Model=Discipline, + Tectonicunittreedef_Model=TectonicTreeDef ) - for discipline in disciplines: - tectonic_tree_def = TectonicTreeDef.objects.filter(discipline=discipline).first() - if not tectonic_tree_def: - tectonic_tree_def, _ = TectonicTreeDef.objects.get_or_create(name="Tectonic Unit", discipline=discipline) + trees_missing_ranks = TectonicTreeDef.objects.filter(treedefitems__isnull=True) + + for tectonic_tree_def in trees_missing_ranks: + + current_parent = None + for default_rank in DEFAULT_RANKS: + # At this point, these get_or_create calls should always be the + # equivalent of create (as we know these nodes didn't exist). + # But keeping the get_or_create here just because + current_parent, _ = TectonicUnitTreeDefItem.objects.get_or_create( + rankid=default_rank["rankid"], + parent=current_parent, + treedef=tectonic_tree_def, + defaults={ + "name": default_rank["name"], + "title": default_rank["name"], + **default_rank.get('attrs', {}) + } + ) + +def create_root_tectonic_node(apps): + TectonicUnit = apps.get_model('specify', 'TectonicUnit') + TectonicUnitTreeDefItem = apps.get_model('specify', 'TectonicUnitTreeDefItem') + TectonicUnitTreeDef = apps.get_model('specify', 'TectonicUnitTreeDef') + + trees_missing_root_node = TectonicUnitTreeDef.objects.annotate( + root_node_exists=Exists( + TectonicUnit.objects.filter( + parent=None, + definition=OuterRef("pk") + ) + ) + ).filter( + root_node_exists=False + ) - root, root_created = TectonicUnitTreeDefItem.objects.get_or_create( + for tree in trees_missing_root_node: + root_rank, _ = TectonicUnitTreeDefItem.objects.get_or_create( rankid=0, parent=None, - treedef=tectonic_tree_def, + treedef=tree, defaults={ "name": "Root", "title": "Root", "isenforced": True } ) - if discipline.tectonicunittreedef_id != tectonic_tree_def.id: - discipline.tectonicunittreedef = tectonic_tree_def - discipline.save(update_fields=["tectonicunittreedef"]) - if not root_created: - # BUG?: handle setting the tectonicunittreedef on the Discipline - # here? We can probably practically assume it's already set if the - # root node exists. - continue - - # At this point, these get_or_create calls should always be the - # equivalent of create (as we know the root node didn't exist). - # But keeping the get_or_create here just because - superstructure, _ = TectonicUnitTreeDefItem.objects.get_or_create( - name="Superstructure", - title="Superstructure", - rankid=10, - parent=root, - treedef=tectonic_tree_def, - ) - tectonic_domain, _ = TectonicUnitTreeDefItem.objects.get_or_create( - name="Tectonic Domain", - title="Tectonic Domain", - rankid=20, - parent=superstructure, - treedef=tectonic_tree_def, - ) - tectonic_subdomain, _ = TectonicUnitTreeDefItem.objects.get_or_create( - name="Tectonic Subdomain", - title="Tectonic Subdomain", - rankid=30, - parent=tectonic_domain, - treedef=tectonic_tree_def, - ) - tectonic_unit, _ = TectonicUnitTreeDefItem.objects.get_or_create( - name="Tectonic Unit", - title="Tectonic Unit", - rankid=40, - parent=tectonic_subdomain, - treedef=tectonic_tree_def, - ) - tectonic_subunit, _ = TectonicUnitTreeDefItem.objects.get_or_create( - name="Tectonic Subunit", - title="Tectonic Subunit", - rankid=50, - parent=tectonic_unit, - treedef=tectonic_tree_def, + TectonicUnit.objects.create( + name="Root", + fullname="Root", + isaccepted=1, + nodenumber=1, + rankid=0, + parent=None, + definition=tree, + definitionitem=root_rank ) + logger.info(f"Created root tectonic unit for discipline {tree.discipline_id}") - discipline.tectonicunittreedef = tectonic_tree_def - discipline.save() - -def revert_default_tectonic_ranks(apps, schema_editor=None): - TectonicUnit = apps.get_model('specify', 'TectonicUnit') - TectonicUnitTreeDefItem = apps.get_model('specify', 'TectonicUnitTreeDefItem') - TectonicTreeDef = apps.get_model('specify', 'TectonicUnitTreeDef') - Discipline = apps.get_model('specify', 'Discipline') - - for discipline in Discipline.objects.all(): - tectonic_tree_defs = TectonicTreeDef.objects.filter(name="Tectonic Unit", discipline=discipline) - - for tectonic_tree_def in tectonic_tree_defs: - tectonic_unit_tree_def_items = TectonicUnitTreeDefItem.objects.filter(treedef=tectonic_tree_def).order_by('-id') - - for item in tectonic_unit_tree_def_items: - TectonicUnit.objects.filter(definitionitem=item).delete() - - item.delete() - - discipline.tectonicunittreedef = None - discipline.save() - tectonic_tree_def.delete() - -def create_root_tectonic_node(apps): - TectonicUnit = apps.get_model('specify', 'TectonicUnit') - TectonicUnitTreeDefItem = apps.get_model('specify', 'TectonicUnitTreeDefItem') - TectonicUnitTreeDef = apps.get_model('specify', 'TectonicUnitTreeDef') - Discipline = apps.get_model('specify', 'Discipline') - - for discipline in Discipline.objects.all(): + TectonicUnitTreeDefItem.objects.filter(parent=None,rankid=0, isenforced__isnull=True).update(isenforced=True) - tectonic_tree_def = TectonicUnitTreeDef.objects.filter(discipline=discipline).first() - if not tectonic_tree_def: - tectonic_tree_def, is_created = TectonicUnitTreeDef.objects.get_or_create( +def _create_tectonic_unit_for_discipline(Discipline_Model, Tectonicunittreedef_Model): + # Fetches Discipline objects with an empty TectonicUnitTreeDef relationship + # and no TectonicUnitTreeDef objects with a set discipline + # Most commonly, this would be in the case of creating a Discipline in + # Specify 6 after the TectonicUnitTreeDef migrations have been run in + # Specify 7 + disciplines_missing_tectonicunit = Discipline_Model.objects.filter( + tectonicunittreedef__isnull=True, + tectonicunittreedefs__isnull=True + ).values_list("pk", flat=True) + + Tectonicunittreedef_Model.objects.bulk_create( + [ + Tectonicunittreedef_Model( name="Tectonic Unit", - discipline=discipline - ) - - tectonic_tree_def_item = TectonicUnitTreeDefItem.objects.filter(treedef=tectonic_tree_def, rankid=0, parent=None).first() - if not tectonic_tree_def_item: - tectonic_tree_def_item, is_created = TectonicUnitTreeDefItem.objects.get_or_create( - name="Root", - title="Root", - treedef=tectonic_tree_def, - rankid=0, - parent=None, - isenforced=True - ) - - root = TectonicUnit.objects.filter(definition=tectonic_tree_def, definitionitem=tectonic_tree_def_item, rankid=0, parent=None).first() - if not root: - root, is_created = TectonicUnit.objects.get_or_create( - name="Root", - fullname="Root", - isaccepted=1, - nodenumber=1, - rankid=0, - parent=None, - definition=tectonic_tree_def, - definitionitem=tectonic_tree_def_item - ) - - if is_created: - logger.info(f"Created root tectonic unit for discipline {discipline.name}") + discipline_id=disciplineid + ) for disciplineid in disciplines_missing_tectonicunit + ], + batch_size=1000 + ) - TectonicUnitTreeDefItem.objects.filter(parent=None,rankid=0, isenforced__isnull=True).update(isenforced=True) + # If there are cases where Discipline -> tectonicunittreedef is not set, + # but there is at least one TectonicUnitTreeDef pointing to the Discipline, + # then set the Discipline -> tectonicunittreedef relationship to the "first" + # TectonicUnitTreeDef -> discipline + Discipline_Model.objects.filter( + tectonicunittreedef__isnull=True, + tectonicunittreedefs__isnull=False + ).update( + tectonicunittreedef=Subquery( + Tectonicunittreedef_Model.objects.filter( + discipline=OuterRef("pk") + ).order_by("pk").values("pk")[:1] + ) + ) -def revert_create_root_tectonic_node(apps, schema_editor=None): - TectonicUnit = apps.get_model('specify', 'TectonicUnit') - TectonicUnitTreeDefItem = apps.get_model('specify', 'TectonicUnitTreeDefItem') - TectonicTreeDef = apps.get_model('specify', 'TectonicUnitTreeDef') +def fix_tectonic_unit_treedef_discipline_links(apps): Discipline = apps.get_model('specify', 'Discipline') + Tectonicunittreedef = apps.get_model('specify', 'Tectonicunittreedef') - for discipline in Discipline.objects.all(): - tectonic_tree_def = TectonicTreeDef.objects.filter(name="Tectonic Unit", discipline=discipline).first() - - if tectonic_tree_def: - TectonicUnitTreeDefItem.objects.filter(treedef=tectonic_tree_def).delete() - TectonicUnit.objects.filter( - name="Root" - ).delete() \ No newline at end of file + _create_tectonic_unit_for_discipline( + Discipline_Model=Discipline, + Tectonicunittreedef_Model=Tectonicunittreedef + ) + + # If there's any TectonicUnitTreeDef objects with a NULL + # discipline, set the discipline relationship to the + # Discipline -> tectonicunittreedef + Tectonicunittreedef.objects.filter( + discipline__isnull=True, + disciplines__isnull=False + ).update( + discipline=Subquery( + Discipline.objects.filter( + tectonicunittreedef=OuterRef("pk") + ).order_by("pk").values("pk")[:1] + ) + ) diff --git a/specifyweb/specify/migration_utils/update_schema_config.py b/specifyweb/specify/migration_utils/update_schema_config.py index b91927b90a6..75591205613 100644 --- a/specifyweb/specify/migration_utils/update_schema_config.py +++ b/specifyweb/specify/migration_utils/update_schema_config.py @@ -410,18 +410,25 @@ def update_table_field_schema_config_with_defaults( Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') - try: - sp_local_container, _ = Splocalecontainer.objects.get_or_create( + sp_local_container = ( + Splocalecontainer.objects.filter( name=table.name.lower(), discipline_id=discipline_id, schematype=table_config.schema_type, ) - except MultipleObjectsReturned: - sp_local_container = Splocalecontainer.objects.filter( + .order_by('id') + .first() + ) + + if sp_local_container is None: + sp_local_container = Splocalecontainer.objects.create( name=table.name.lower(), discipline_id=discipline_id, - schematype=table_config.schema_type - ).first() + schematype=table_config.schema_type, + ishidden=False, + issystem=table.system, + version=0, + ) try: field = table.get_field_strict(field_name) @@ -535,18 +542,25 @@ def update_table_field_schema_config_params( Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') - try: - sp_local_container, _ = Splocalecontainer.objects.get_or_create( + sp_local_container = ( + Splocalecontainer.objects.filter( name=table.name.lower(), discipline_id=discipline_id, schematype=table_config.schema_type, ) - except MultipleObjectsReturned: - sp_local_container = Splocalecontainer.objects.filter( + .order_by('id') + .first() + ) + + if sp_local_container is None: + sp_local_container = Splocalecontainer.objects.create( name=table.name.lower(), discipline_id=discipline_id, - schematype=table_config.schema_type - ).first() + schematype=table_config.schema_type, + ishidden=False, + issystem=table.system, + version=0, + ) try: field = table.get_field_strict(field_name) @@ -968,7 +982,7 @@ def update_cog_type_fields(apps): container_items = Splocalecontaineritem.objects.filter( name="collectionObjectType", picklistname=None, - container__name="CollectionObject", + container__name="collectionobject", ) for container_item in container_items: Splocaleitemstr.objects.filter(itemname=container_item).delete() @@ -1383,7 +1397,7 @@ def update_schema_config_field_desc(apps, schema_editor=None): #i.e: COType items = Splocalecontaineritem.objects.filter( container=container, - name=field_name.lower() + name__iexact=field_name ) for item in items: @@ -1585,7 +1599,7 @@ def update_schema_config_field_desc(apps): #i.e: COType items = Splocalecontaineritem.objects.filter( container=container, - name=field_name.lower() + name__iexact=field_name ) for item in items: @@ -1661,7 +1675,7 @@ def update_schema_config_field_desc(apps): #i.e: COType items = Splocalecontaineritem.objects.filter( container=container, - name=field_name.lower() + name__iexact=field_name ) for item in items: @@ -1787,7 +1801,7 @@ def update_schema_config_field_desc(apps): #i.e: COType items = Splocalecontaineritem.objects.filter( container=container, - name=field_name.lower() + name__iexact=field_name ) for item in items: @@ -1880,19 +1894,19 @@ def update_0034_schema_config_field_desc(apps): for (field_name, new_name, new_desc) in fields: items = Splocalecontaineritem.objects.filter( container=container, - name=field_name.lower() + name__iexact=field_name ) for item in items: item.ishidden = True item.save() desc_str = Splocaleitemstr.objects.filter(itemdesc_id=item.id).first() name_str = Splocaleitemstr.objects.filter(itemname_id=item.id).first() - if not desc_str or not name_str: - continue - desc_str.text = new_desc - desc_str.save() - name_str.text = new_name - name_str.save() + if desc_str is not None: + desc_str.text = new_desc + desc_str.save() + if name_str is not None: + name_str.text = new_name + name_str.save() update_0034_fields(apps) update_0034_schema_config_field_desc(apps) @@ -1919,7 +1933,7 @@ def revert_0034_schema_config_field_desc(apps): for (field_name, _, _) in fields: items = Splocalecontaineritem.objects.filter( container=container, - name=field_name.lower() + name__iexact=field_name ) # If needed, reset ishidden or revert text @@ -2019,13 +2033,15 @@ def update_schema_config_field_desc_for_components(apps, schema_editor=None): Splocaleitemstr.objects.filter( itemdesc__container__name=table.lower(), itemdesc__container__schematype=0, - itemdesc__name=field_name.lower() + itemdesc__name=field_name.lower(), + language="en", ).update(text=new_desc) Splocaleitemstr.objects.filter( itemname__container__name=table.lower(), itemname__container__schematype=0, - itemname__name=field_name.lower() + itemname__name=field_name.lower(), + language="en", ).update(text=new_name) def update_hidden_prop_for_compoenents(apps, schema_editor=None): @@ -2097,7 +2113,7 @@ def reverse_hide_component_fields(apps, schema_editor=None): container=container, name=field_name.lower() ) - items.update(ishidden=True) + items.update(ishidden=False) # ########################################## # Used in 0042_discipline_type_picklist.py diff --git a/specifyweb/specify/migrations/0002_geo.py b/specifyweb/specify/migrations/0002_geo.py index 58cffd3a12f..e84e59d5c2e 100644 --- a/specifyweb/specify/migrations/0002_geo.py +++ b/specifyweb/specify/migrations/0002_geo.py @@ -16,7 +16,6 @@ create_default_discipline_for_tree_defs, set_discipline_for_taxon_treedefs, ) -from specifyweb.specify.api.utils import create_default_collection_types logger = logging.getLogger(__name__) @@ -72,12 +71,11 @@ class Migration(migrations.Migration): ] def consolidated_python_django_migration_operations(apps, schema_editor): - db_alias = schema_editor.connection.alias or 'migrator' - create_default_collection_types(apps, using=db_alias) - create_default_discipline_for_tree_defs(apps, using=db_alias) + create_default_collection_types(apps) + create_default_discipline_for_tree_defs(apps) usc.create_geo_table_schema_config_with_defaults(apps) - create_cogtype_type_picklist(apps, using=db_alias) - set_discipline_for_taxon_treedefs(apps, using=db_alias) + create_cogtype_type_picklist(apps) + set_discipline_for_taxon_treedefs(apps) def revert_cosolidated_python_django_migration_operations(apps, schema_editor): revert_cogtype_type_picklist(apps) diff --git a/specifyweb/specify/migrations/0008_ageCitations_fix.py b/specifyweb/specify/migrations/0008_ageCitations_fix.py index 479d9dae005..9f058e2dbf3 100644 --- a/specifyweb/specify/migrations/0008_ageCitations_fix.py +++ b/specifyweb/specify/migrations/0008_ageCitations_fix.py @@ -15,7 +15,7 @@ def apply_migration(apps, schema_editor): usc.update_relative_age_fields(apps) def revert_migration(apps, schema_editor): - usc.update_relative_age_fields(apps) + usc.revert_relative_age_fields(apps) operations = [ migrations.AddField( diff --git a/specifyweb/specify/migrations/0009_tectonic_ranks.py b/specifyweb/specify/migrations/0009_tectonic_ranks.py index 54182060ebe..975fb3570b1 100644 --- a/specifyweb/specify/migrations/0009_tectonic_ranks.py +++ b/specifyweb/specify/migrations/0009_tectonic_ranks.py @@ -1,12 +1,108 @@ from django.db import migrations +from django.db.models import Exists, OuterRef, Subquery from specifyweb.specify.migration_utils.tectonic_ranks import ( create_default_tectonic_ranks, create_root_tectonic_node, - revert_create_root_tectonic_node, - revert_default_tectonic_ranks, + DEFAULT_RANKS ) +def revert_create_root_tectonic_node(apps, schema_editor=None): + TectonicUnit = apps.get_model('specify', 'TectonicUnit') + TectonicUnitTreeDefItem = apps.get_model('specify', 'TectonicUnitTreeDefItem') + + # Technically at this point a user could have more than just the root node + # in the tree, so only delete the TectonicUnit nodes which were created + # from create_root_tectonic_node and are alone in the tree + TectonicUnit.objects.annotate( + has_children_nodes=Exists( + TectonicUnit.objects.filter( + parent=OuterRef("pk") + ) + ) + ).filter( + parent__isnull=True, + has_children_nodes=False, + acceptedtectonicunit__isnull=True, + name="Root" + ).delete() + + # Delete the Root TectonicUnit rank if there are no nodes in the tree and + # no children rank reference the Root rank + TectonicUnitTreeDefItem.objects.annotate( + has_nodes=Exists( + TectonicUnit.objects.filter( + definitionitem=OuterRef("pk") + ) + ), + has_child_rank=Exists( + TectonicUnitTreeDefItem.objects.filter( + parent=OuterRef("pk") + ) + ) + ).filter( + has_nodes=False, + has_child_rank=False, + name="Root" + ).delete() + +# REFACTOR: Optimize this +def revert_default_tectonic_ranks(apps, schema_editor=None): + TectonicUnit = apps.get_model('specify', 'TectonicUnit') + TectonicUnitTreeDefItem = apps.get_model('specify', 'TectonicUnitTreeDefItem') + TectonicTreeDef = apps.get_model('specify', 'TectonicUnitTreeDef') + Discipline = apps.get_model('specify', 'Discipline') + + for default_rank in reversed(DEFAULT_RANKS): + tree_def_items = TectonicUnitTreeDefItem.objects.annotate( + has_child_rank=Exists( + TectonicUnitTreeDefItem.objects.filter( + parent=OuterRef("pk") + ) + ) + ).filter( + has_child_rank=False, + name=default_rank["name"], + # rankid=default_rank["rankid"] + ) + + units_to_delete = TectonicUnit.objects.annotate( + has_children_nodes=Exists( + TectonicUnit.objects.filter( + parent=OuterRef("pk") + ) + ) + ).filter( + has_children_nodes=False, + definitionitem__in=tree_def_items + ) + TectonicUnit.objects.filter( + acceptedtectonicunit_id__in=units_to_delete.values_list('pk', flat=True) + ).update( + acceptedtectonicunit=None, + isaccepted=True + ) + + units_to_delete.delete() + tree_def_items.delete() + + empty_tree_defs = TectonicTreeDef.objects.annotate( + has_ranks=Exists( + TectonicUnitTreeDefItem.objects.filter( + treedef=OuterRef("pk") + ) + ) + ).filter( + has_ranks=False + ) + + Discipline.objects.filter( + tectonicunittreedef_id__in=empty_tree_defs.values_list('pk', flat=True) + ).update( + tectonicunittreedef=None + ) + empty_tree_defs.delete() + class Migration(migrations.Migration): dependencies = [ @@ -18,8 +114,8 @@ def consolidated_python_django_migration_operations(apps, schema_editor): create_root_tectonic_node(apps) def revert_cosolidated_python_django_migration_operations(apps, schema_editor): - revert_default_tectonic_ranks(apps, schema_editor) revert_create_root_tectonic_node(apps, schema_editor) + revert_default_tectonic_ranks(apps, schema_editor) operations = [ migrations.RunPython(