schema.py 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493
  1. import copy
  2. from decimal import Decimal
  3. from django.apps.registry import Apps
  4. from django.db import NotSupportedError
  5. from django.db.backends.base.schema import BaseDatabaseSchemaEditor
  6. from django.db.backends.ddl_references import Statement
  7. from django.db.backends.utils import strip_quotes
  8. from django.db.models import NOT_PROVIDED, UniqueConstraint
  9. class DatabaseSchemaEditor(BaseDatabaseSchemaEditor):
  10. sql_delete_table = "DROP TABLE %(table)s"
  11. sql_create_fk = None
  12. sql_create_inline_fk = (
  13. "REFERENCES %(to_table)s (%(to_column)s) DEFERRABLE INITIALLY DEFERRED"
  14. )
  15. sql_create_column_inline_fk = sql_create_inline_fk
  16. sql_delete_column = "ALTER TABLE %(table)s DROP COLUMN %(column)s"
  17. sql_create_unique = "CREATE UNIQUE INDEX %(name)s ON %(table)s (%(columns)s)"
  18. sql_delete_unique = "DROP INDEX %(name)s"
  19. sql_alter_table_comment = None
  20. sql_alter_column_comment = None
  21. def __enter__(self):
  22. # Some SQLite schema alterations need foreign key constraints to be
  23. # disabled. Enforce it here for the duration of the schema edition.
  24. if not self.connection.disable_constraint_checking():
  25. raise NotSupportedError(
  26. "SQLite schema editor cannot be used while foreign key "
  27. "constraint checks are enabled. Make sure to disable them "
  28. "before entering a transaction.atomic() context because "
  29. "SQLite does not support disabling them in the middle of "
  30. "a multi-statement transaction."
  31. )
  32. return super().__enter__()
  33. def __exit__(self, exc_type, exc_value, traceback):
  34. self.connection.check_constraints()
  35. super().__exit__(exc_type, exc_value, traceback)
  36. self.connection.enable_constraint_checking()
  37. def quote_value(self, value):
  38. # The backend "mostly works" without this function and there are use
  39. # cases for compiling Python without the sqlite3 libraries (e.g.
  40. # security hardening).
  41. try:
  42. import sqlite3
  43. value = sqlite3.adapt(value)
  44. except ImportError:
  45. pass
  46. except sqlite3.ProgrammingError:
  47. pass
  48. # Manual emulation of SQLite parameter quoting
  49. if isinstance(value, bool):
  50. return str(int(value))
  51. elif isinstance(value, (Decimal, float, int)):
  52. return str(value)
  53. elif isinstance(value, str):
  54. return "'%s'" % value.replace("'", "''")
  55. elif value is None:
  56. return "NULL"
  57. elif isinstance(value, (bytes, bytearray, memoryview)):
  58. # Bytes are only allowed for BLOB fields, encoded as string
  59. # literals containing hexadecimal data and preceded by a single "X"
  60. # character.
  61. return "X'%s'" % value.hex()
  62. else:
  63. raise ValueError(
  64. "Cannot quote parameter value %r of type %s" % (value, type(value))
  65. )
  66. def prepare_default(self, value):
  67. return self.quote_value(value)
  68. def _remake_table(
  69. self, model, create_field=None, delete_field=None, alter_fields=None
  70. ):
  71. """
  72. Shortcut to transform a model from old_model into new_model
  73. This follows the correct procedure to perform non-rename or column
  74. addition operations based on SQLite's documentation
  75. https://www.sqlite.org/lang_altertable.html#caution
  76. The essential steps are:
  77. 1. Create a table with the updated definition called "new__app_model"
  78. 2. Copy the data from the existing "app_model" table to the new table
  79. 3. Drop the "app_model" table
  80. 4. Rename the "new__app_model" table to "app_model"
  81. 5. Restore any index of the previous "app_model" table.
  82. """
  83. # Self-referential fields must be recreated rather than copied from
  84. # the old model to ensure their remote_field.field_name doesn't refer
  85. # to an altered field.
  86. def is_self_referential(f):
  87. return f.is_relation and f.remote_field.model is model
  88. # Work out the new fields dict / mapping
  89. body = {
  90. f.name: f.clone() if is_self_referential(f) else f
  91. for f in model._meta.local_concrete_fields
  92. }
  93. # Since mapping might mix column names and default values,
  94. # its values must be already quoted.
  95. mapping = {
  96. f.column: self.quote_name(f.column)
  97. for f in model._meta.local_concrete_fields
  98. if f.generated is False
  99. }
  100. # This maps field names (not columns) for things like unique_together
  101. rename_mapping = {}
  102. # If any of the new or altered fields is introducing a new PK,
  103. # remove the old one
  104. restore_pk_field = None
  105. alter_fields = alter_fields or []
  106. if getattr(create_field, "primary_key", False) or any(
  107. getattr(new_field, "primary_key", False) for _, new_field in alter_fields
  108. ):
  109. for name, field in list(body.items()):
  110. if field.primary_key and not any(
  111. # Do not remove the old primary key when an altered field
  112. # that introduces a primary key is the same field.
  113. name == new_field.name
  114. for _, new_field in alter_fields
  115. ):
  116. field.primary_key = False
  117. restore_pk_field = field
  118. if field.auto_created:
  119. del body[name]
  120. del mapping[field.column]
  121. # Add in any created fields
  122. if create_field:
  123. body[create_field.name] = create_field
  124. # Choose a default and insert it into the copy map
  125. if (
  126. create_field.db_default is NOT_PROVIDED
  127. and not (create_field.many_to_many or create_field.generated)
  128. and create_field.concrete
  129. ):
  130. mapping[create_field.column] = self.prepare_default(
  131. self.effective_default(create_field)
  132. )
  133. # Add in any altered fields
  134. for alter_field in alter_fields:
  135. old_field, new_field = alter_field
  136. body.pop(old_field.name, None)
  137. mapping.pop(old_field.column, None)
  138. body[new_field.name] = new_field
  139. if old_field.null and not new_field.null:
  140. if new_field.db_default is NOT_PROVIDED:
  141. default = self.prepare_default(self.effective_default(new_field))
  142. else:
  143. default, _ = self.db_default_sql(new_field)
  144. case_sql = "coalesce(%(col)s, %(default)s)" % {
  145. "col": self.quote_name(old_field.column),
  146. "default": default,
  147. }
  148. mapping[new_field.column] = case_sql
  149. else:
  150. mapping[new_field.column] = self.quote_name(old_field.column)
  151. rename_mapping[old_field.name] = new_field.name
  152. # Remove any deleted fields
  153. if delete_field:
  154. del body[delete_field.name]
  155. mapping.pop(delete_field.column, None)
  156. # Remove any implicit M2M tables
  157. if (
  158. delete_field.many_to_many
  159. and delete_field.remote_field.through._meta.auto_created
  160. ):
  161. return self.delete_model(delete_field.remote_field.through)
  162. # Work inside a new app registry
  163. apps = Apps()
  164. # Work out the new value of unique_together, taking renames into
  165. # account
  166. unique_together = [
  167. [rename_mapping.get(n, n) for n in unique]
  168. for unique in model._meta.unique_together
  169. ]
  170. # RemovedInDjango51Warning.
  171. # Work out the new value for index_together, taking renames into
  172. # account
  173. index_together = [
  174. [rename_mapping.get(n, n) for n in index]
  175. for index in model._meta.index_together
  176. ]
  177. indexes = model._meta.indexes
  178. if delete_field:
  179. indexes = [
  180. index for index in indexes if delete_field.name not in index.fields
  181. ]
  182. constraints = list(model._meta.constraints)
  183. # Provide isolated instances of the fields to the new model body so
  184. # that the existing model's internals aren't interfered with when
  185. # the dummy model is constructed.
  186. body_copy = copy.deepcopy(body)
  187. # Construct a new model with the new fields to allow self referential
  188. # primary key to resolve to. This model won't ever be materialized as a
  189. # table and solely exists for foreign key reference resolution purposes.
  190. # This wouldn't be required if the schema editor was operating on model
  191. # states instead of rendered models.
  192. meta_contents = {
  193. "app_label": model._meta.app_label,
  194. "db_table": model._meta.db_table,
  195. "unique_together": unique_together,
  196. "index_together": index_together, # RemovedInDjango51Warning.
  197. "indexes": indexes,
  198. "constraints": constraints,
  199. "apps": apps,
  200. }
  201. meta = type("Meta", (), meta_contents)
  202. body_copy["Meta"] = meta
  203. body_copy["__module__"] = model.__module__
  204. type(model._meta.object_name, model.__bases__, body_copy)
  205. # Construct a model with a renamed table name.
  206. body_copy = copy.deepcopy(body)
  207. meta_contents = {
  208. "app_label": model._meta.app_label,
  209. "db_table": "new__%s" % strip_quotes(model._meta.db_table),
  210. "unique_together": unique_together,
  211. "index_together": index_together, # RemovedInDjango51Warning.
  212. "indexes": indexes,
  213. "constraints": constraints,
  214. "apps": apps,
  215. }
  216. meta = type("Meta", (), meta_contents)
  217. body_copy["Meta"] = meta
  218. body_copy["__module__"] = model.__module__
  219. new_model = type("New%s" % model._meta.object_name, model.__bases__, body_copy)
  220. # Create a new table with the updated schema.
  221. self.create_model(new_model)
  222. # Copy data from the old table into the new table
  223. self.execute(
  224. "INSERT INTO %s (%s) SELECT %s FROM %s"
  225. % (
  226. self.quote_name(new_model._meta.db_table),
  227. ", ".join(self.quote_name(x) for x in mapping),
  228. ", ".join(mapping.values()),
  229. self.quote_name(model._meta.db_table),
  230. )
  231. )
  232. # Delete the old table to make way for the new
  233. self.delete_model(model, handle_autom2m=False)
  234. # Rename the new table to take way for the old
  235. self.alter_db_table(
  236. new_model,
  237. new_model._meta.db_table,
  238. model._meta.db_table,
  239. )
  240. # Run deferred SQL on correct table
  241. for sql in self.deferred_sql:
  242. self.execute(sql)
  243. self.deferred_sql = []
  244. # Fix any PK-removed field
  245. if restore_pk_field:
  246. restore_pk_field.primary_key = True
  247. def delete_model(self, model, handle_autom2m=True):
  248. if handle_autom2m:
  249. super().delete_model(model)
  250. else:
  251. # Delete the table (and only that)
  252. self.execute(
  253. self.sql_delete_table
  254. % {
  255. "table": self.quote_name(model._meta.db_table),
  256. }
  257. )
  258. # Remove all deferred statements referencing the deleted table.
  259. for sql in list(self.deferred_sql):
  260. if isinstance(sql, Statement) and sql.references_table(
  261. model._meta.db_table
  262. ):
  263. self.deferred_sql.remove(sql)
  264. def add_field(self, model, field):
  265. """Create a field on a model."""
  266. from django.db.models.expressions import Value
  267. # Special-case implicit M2M tables.
  268. if field.many_to_many and field.remote_field.through._meta.auto_created:
  269. self.create_model(field.remote_field.through)
  270. elif (
  271. # Primary keys and unique fields are not supported in ALTER TABLE
  272. # ADD COLUMN.
  273. field.primary_key
  274. or field.unique
  275. or not field.null
  276. # Fields with default values cannot by handled by ALTER TABLE ADD
  277. # COLUMN statement because DROP DEFAULT is not supported in
  278. # ALTER TABLE.
  279. or self.effective_default(field) is not None
  280. # Fields with non-constant defaults cannot by handled by ALTER
  281. # TABLE ADD COLUMN statement.
  282. or (
  283. field.db_default is not NOT_PROVIDED
  284. and not isinstance(field.db_default, Value)
  285. )
  286. ):
  287. self._remake_table(model, create_field=field)
  288. else:
  289. super().add_field(model, field)
  290. def remove_field(self, model, field):
  291. """
  292. Remove a field from a model. Usually involves deleting a column,
  293. but for M2Ms may involve deleting a table.
  294. """
  295. # M2M fields are a special case
  296. if field.many_to_many:
  297. # For implicit M2M tables, delete the auto-created table
  298. if field.remote_field.through._meta.auto_created:
  299. self.delete_model(field.remote_field.through)
  300. # For explicit "through" M2M fields, do nothing
  301. elif (
  302. self.connection.features.can_alter_table_drop_column
  303. # Primary keys, unique fields, indexed fields, and foreign keys are
  304. # not supported in ALTER TABLE DROP COLUMN.
  305. and not field.primary_key
  306. and not field.unique
  307. and not field.db_index
  308. and not (field.remote_field and field.db_constraint)
  309. ):
  310. super().remove_field(model, field)
  311. # For everything else, remake.
  312. else:
  313. # It might not actually have a column behind it
  314. if field.db_parameters(connection=self.connection)["type"] is None:
  315. return
  316. self._remake_table(model, delete_field=field)
  317. def _alter_field(
  318. self,
  319. model,
  320. old_field,
  321. new_field,
  322. old_type,
  323. new_type,
  324. old_db_params,
  325. new_db_params,
  326. strict=False,
  327. ):
  328. """Perform a "physical" (non-ManyToMany) field update."""
  329. # Use "ALTER TABLE ... RENAME COLUMN" if only the column name
  330. # changed and there aren't any constraints.
  331. if (
  332. old_field.column != new_field.column
  333. and self.column_sql(model, old_field) == self.column_sql(model, new_field)
  334. and not (
  335. old_field.remote_field
  336. and old_field.db_constraint
  337. or new_field.remote_field
  338. and new_field.db_constraint
  339. )
  340. ):
  341. return self.execute(
  342. self._rename_field_sql(
  343. model._meta.db_table, old_field, new_field, new_type
  344. )
  345. )
  346. # Alter by remaking table
  347. self._remake_table(model, alter_fields=[(old_field, new_field)])
  348. # Rebuild tables with FKs pointing to this field.
  349. old_collation = old_db_params.get("collation")
  350. new_collation = new_db_params.get("collation")
  351. if new_field.unique and (
  352. old_type != new_type or old_collation != new_collation
  353. ):
  354. related_models = set()
  355. opts = new_field.model._meta
  356. for remote_field in opts.related_objects:
  357. # Ignore self-relationship since the table was already rebuilt.
  358. if remote_field.related_model == model:
  359. continue
  360. if not remote_field.many_to_many:
  361. if remote_field.field_name == new_field.name:
  362. related_models.add(remote_field.related_model)
  363. elif new_field.primary_key and remote_field.through._meta.auto_created:
  364. related_models.add(remote_field.through)
  365. if new_field.primary_key:
  366. for many_to_many in opts.many_to_many:
  367. # Ignore self-relationship since the table was already rebuilt.
  368. if many_to_many.related_model == model:
  369. continue
  370. if many_to_many.remote_field.through._meta.auto_created:
  371. related_models.add(many_to_many.remote_field.through)
  372. for related_model in related_models:
  373. self._remake_table(related_model)
  374. def _alter_many_to_many(self, model, old_field, new_field, strict):
  375. """Alter M2Ms to repoint their to= endpoints."""
  376. if (
  377. old_field.remote_field.through._meta.db_table
  378. == new_field.remote_field.through._meta.db_table
  379. ):
  380. # The field name didn't change, but some options did, so we have to
  381. # propagate this altering.
  382. self._remake_table(
  383. old_field.remote_field.through,
  384. alter_fields=[
  385. (
  386. # The field that points to the target model is needed,
  387. # so that table can be remade with the new m2m field -
  388. # this is m2m_reverse_field_name().
  389. old_field.remote_field.through._meta.get_field(
  390. old_field.m2m_reverse_field_name()
  391. ),
  392. new_field.remote_field.through._meta.get_field(
  393. new_field.m2m_reverse_field_name()
  394. ),
  395. ),
  396. (
  397. # The field that points to the model itself is needed,
  398. # so that table can be remade with the new self field -
  399. # this is m2m_field_name().
  400. old_field.remote_field.through._meta.get_field(
  401. old_field.m2m_field_name()
  402. ),
  403. new_field.remote_field.through._meta.get_field(
  404. new_field.m2m_field_name()
  405. ),
  406. ),
  407. ],
  408. )
  409. return
  410. # Make a new through table
  411. self.create_model(new_field.remote_field.through)
  412. # Copy the data across
  413. self.execute(
  414. "INSERT INTO %s (%s) SELECT %s FROM %s"
  415. % (
  416. self.quote_name(new_field.remote_field.through._meta.db_table),
  417. ", ".join(
  418. [
  419. "id",
  420. new_field.m2m_column_name(),
  421. new_field.m2m_reverse_name(),
  422. ]
  423. ),
  424. ", ".join(
  425. [
  426. "id",
  427. old_field.m2m_column_name(),
  428. old_field.m2m_reverse_name(),
  429. ]
  430. ),
  431. self.quote_name(old_field.remote_field.through._meta.db_table),
  432. )
  433. )
  434. # Delete the old through table
  435. self.delete_model(old_field.remote_field.through)
  436. def add_constraint(self, model, constraint):
  437. if isinstance(constraint, UniqueConstraint) and (
  438. constraint.condition
  439. or constraint.contains_expressions
  440. or constraint.include
  441. or constraint.deferrable
  442. ):
  443. super().add_constraint(model, constraint)
  444. else:
  445. self._remake_table(model)
  446. def remove_constraint(self, model, constraint):
  447. if isinstance(constraint, UniqueConstraint) and (
  448. constraint.condition
  449. or constraint.contains_expressions
  450. or constraint.include
  451. or constraint.deferrable
  452. ):
  453. super().remove_constraint(model, constraint)
  454. else:
  455. self._remake_table(model)
  456. def _collate_sql(self, collation):
  457. return "COLLATE " + collation