base.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452
  1. """
  2. MySQL database backend for Django.
  3. Requires mysqlclient: https://pypi.org/project/mysqlclient/
  4. """
  5. from django.core.exceptions import ImproperlyConfigured
  6. from django.db import IntegrityError
  7. from django.db.backends import utils as backend_utils
  8. from django.db.backends.base.base import BaseDatabaseWrapper
  9. from django.utils.asyncio import async_unsafe
  10. from django.utils.functional import cached_property
  11. from django.utils.regex_helper import _lazy_re_compile
  12. try:
  13. import MySQLdb as Database
  14. except ImportError as err:
  15. raise ImproperlyConfigured(
  16. "Error loading MySQLdb module.\nDid you install mysqlclient?"
  17. ) from err
  18. from MySQLdb.constants import CLIENT, FIELD_TYPE
  19. from MySQLdb.converters import conversions
  20. # Some of these import MySQLdb, so import them after checking if it's installed.
  21. from .client import DatabaseClient
  22. from .creation import DatabaseCreation
  23. from .features import DatabaseFeatures
  24. from .introspection import DatabaseIntrospection
  25. from .operations import DatabaseOperations
  26. from .schema import DatabaseSchemaEditor
  27. from .validation import DatabaseValidation
  28. version = Database.version_info
  29. if version < (1, 4, 3):
  30. raise ImproperlyConfigured(
  31. "mysqlclient 1.4.3 or newer is required; you have %s." % Database.__version__
  32. )
  33. # MySQLdb returns TIME columns as timedelta -- they are more like timedelta in
  34. # terms of actual behavior as they are signed and include days -- and Django
  35. # expects time.
  36. django_conversions = {
  37. **conversions,
  38. **{FIELD_TYPE.TIME: backend_utils.typecast_time},
  39. }
  40. # This should match the numerical portion of the version numbers (we can treat
  41. # versions like 5.0.24 and 5.0.24a as the same).
  42. server_version_re = _lazy_re_compile(r"(\d{1,2})\.(\d{1,2})\.(\d{1,2})")
  43. class CursorWrapper:
  44. """
  45. A thin wrapper around MySQLdb's normal cursor class that catches particular
  46. exception instances and reraises them with the correct types.
  47. Implemented as a wrapper, rather than a subclass, so that it isn't stuck
  48. to the particular underlying representation returned by Connection.cursor().
  49. """
  50. codes_for_integrityerror = (
  51. 1048, # Column cannot be null
  52. 1690, # BIGINT UNSIGNED value is out of range
  53. 3819, # CHECK constraint is violated
  54. 4025, # CHECK constraint failed
  55. )
  56. def __init__(self, cursor):
  57. self.cursor = cursor
  58. def execute(self, query, args=None):
  59. try:
  60. # args is None means no string interpolation
  61. return self.cursor.execute(query, args)
  62. except Database.OperationalError as e:
  63. # Map some error codes to IntegrityError, since they seem to be
  64. # misclassified and Django would prefer the more logical place.
  65. if e.args[0] in self.codes_for_integrityerror:
  66. raise IntegrityError(*tuple(e.args))
  67. raise
  68. def executemany(self, query, args):
  69. try:
  70. return self.cursor.executemany(query, args)
  71. except Database.OperationalError as e:
  72. # Map some error codes to IntegrityError, since they seem to be
  73. # misclassified and Django would prefer the more logical place.
  74. if e.args[0] in self.codes_for_integrityerror:
  75. raise IntegrityError(*tuple(e.args))
  76. raise
  77. def __getattr__(self, attr):
  78. return getattr(self.cursor, attr)
  79. def __iter__(self):
  80. return iter(self.cursor)
  81. class DatabaseWrapper(BaseDatabaseWrapper):
  82. vendor = "mysql"
  83. # This dictionary maps Field objects to their associated MySQL column
  84. # types, as strings. Column-type strings can contain format strings; they'll
  85. # be interpolated against the values of Field.__dict__ before being output.
  86. # If a column type is set to None, it won't be included in the output.
  87. _data_types = {
  88. "AutoField": "integer AUTO_INCREMENT",
  89. "BigAutoField": "bigint AUTO_INCREMENT",
  90. "BinaryField": "longblob",
  91. "BooleanField": "bool",
  92. "CharField": "varchar(%(max_length)s)",
  93. "DateField": "date",
  94. "DateTimeField": "datetime(6)",
  95. "DecimalField": "numeric(%(max_digits)s, %(decimal_places)s)",
  96. "DurationField": "bigint",
  97. "FileField": "varchar(%(max_length)s)",
  98. "FilePathField": "varchar(%(max_length)s)",
  99. "FloatField": "double precision",
  100. "IntegerField": "integer",
  101. "BigIntegerField": "bigint",
  102. "IPAddressField": "char(15)",
  103. "GenericIPAddressField": "char(39)",
  104. "JSONField": "json",
  105. "OneToOneField": "integer",
  106. "PositiveBigIntegerField": "bigint UNSIGNED",
  107. "PositiveIntegerField": "integer UNSIGNED",
  108. "PositiveSmallIntegerField": "smallint UNSIGNED",
  109. "SlugField": "varchar(%(max_length)s)",
  110. "SmallAutoField": "smallint AUTO_INCREMENT",
  111. "SmallIntegerField": "smallint",
  112. "TextField": "longtext",
  113. "TimeField": "time(6)",
  114. "UUIDField": "char(32)",
  115. }
  116. @cached_property
  117. def data_types(self):
  118. _data_types = self._data_types.copy()
  119. if self.features.has_native_uuid_field:
  120. _data_types["UUIDField"] = "uuid"
  121. return _data_types
  122. # For these data types:
  123. # - MySQL < 8.0.13 doesn't accept default values and implicitly treats them
  124. # as nullable
  125. # - all versions of MySQL and MariaDB don't support full width database
  126. # indexes
  127. _limited_data_types = (
  128. "tinyblob",
  129. "blob",
  130. "mediumblob",
  131. "longblob",
  132. "tinytext",
  133. "text",
  134. "mediumtext",
  135. "longtext",
  136. "json",
  137. )
  138. operators = {
  139. "exact": "= %s",
  140. "iexact": "LIKE %s",
  141. "contains": "LIKE BINARY %s",
  142. "icontains": "LIKE %s",
  143. "gt": "> %s",
  144. "gte": ">= %s",
  145. "lt": "< %s",
  146. "lte": "<= %s",
  147. "startswith": "LIKE BINARY %s",
  148. "endswith": "LIKE BINARY %s",
  149. "istartswith": "LIKE %s",
  150. "iendswith": "LIKE %s",
  151. }
  152. # The patterns below are used to generate SQL pattern lookup clauses when
  153. # the right-hand side of the lookup isn't a raw string (it might be an expression
  154. # or the result of a bilateral transformation).
  155. # In those cases, special characters for LIKE operators (e.g. \, *, _) should be
  156. # escaped on database side.
  157. #
  158. # Note: we use str.format() here for readability as '%' is used as a wildcard for
  159. # the LIKE operator.
  160. pattern_esc = r"REPLACE(REPLACE(REPLACE({}, '\\', '\\\\'), '%%', '\%%'), '_', '\_')"
  161. pattern_ops = {
  162. "contains": "LIKE BINARY CONCAT('%%', {}, '%%')",
  163. "icontains": "LIKE CONCAT('%%', {}, '%%')",
  164. "startswith": "LIKE BINARY CONCAT({}, '%%')",
  165. "istartswith": "LIKE CONCAT({}, '%%')",
  166. "endswith": "LIKE BINARY CONCAT('%%', {})",
  167. "iendswith": "LIKE CONCAT('%%', {})",
  168. }
  169. isolation_levels = {
  170. "read uncommitted",
  171. "read committed",
  172. "repeatable read",
  173. "serializable",
  174. }
  175. Database = Database
  176. SchemaEditorClass = DatabaseSchemaEditor
  177. # Classes instantiated in __init__().
  178. client_class = DatabaseClient
  179. creation_class = DatabaseCreation
  180. features_class = DatabaseFeatures
  181. introspection_class = DatabaseIntrospection
  182. ops_class = DatabaseOperations
  183. validation_class = DatabaseValidation
  184. def get_database_version(self):
  185. return self.mysql_version
  186. def get_connection_params(self):
  187. kwargs = {
  188. "conv": django_conversions,
  189. "charset": "utf8",
  190. }
  191. settings_dict = self.settings_dict
  192. if settings_dict["USER"]:
  193. kwargs["user"] = settings_dict["USER"]
  194. if settings_dict["NAME"]:
  195. kwargs["database"] = settings_dict["NAME"]
  196. if settings_dict["PASSWORD"]:
  197. kwargs["password"] = settings_dict["PASSWORD"]
  198. if settings_dict["HOST"].startswith("/"):
  199. kwargs["unix_socket"] = settings_dict["HOST"]
  200. elif settings_dict["HOST"]:
  201. kwargs["host"] = settings_dict["HOST"]
  202. if settings_dict["PORT"]:
  203. kwargs["port"] = int(settings_dict["PORT"])
  204. # We need the number of potentially affected rows after an
  205. # "UPDATE", not the number of changed rows.
  206. kwargs["client_flag"] = CLIENT.FOUND_ROWS
  207. # Validate the transaction isolation level, if specified.
  208. options = settings_dict["OPTIONS"].copy()
  209. isolation_level = options.pop("isolation_level", "read committed")
  210. if isolation_level:
  211. isolation_level = isolation_level.lower()
  212. if isolation_level not in self.isolation_levels:
  213. raise ImproperlyConfigured(
  214. "Invalid transaction isolation level '%s' specified.\n"
  215. "Use one of %s, or None."
  216. % (
  217. isolation_level,
  218. ", ".join("'%s'" % s for s in sorted(self.isolation_levels)),
  219. )
  220. )
  221. self.isolation_level = isolation_level
  222. kwargs.update(options)
  223. return kwargs
  224. @async_unsafe
  225. def get_new_connection(self, conn_params):
  226. connection = Database.connect(**conn_params)
  227. # bytes encoder in mysqlclient doesn't work and was added only to
  228. # prevent KeyErrors in Django < 2.0. We can remove this workaround when
  229. # mysqlclient 2.1 becomes the minimal mysqlclient supported by Django.
  230. # See https://github.com/PyMySQL/mysqlclient/issues/489
  231. if connection.encoders.get(bytes) is bytes:
  232. connection.encoders.pop(bytes)
  233. return connection
  234. def init_connection_state(self):
  235. super().init_connection_state()
  236. assignments = []
  237. if self.features.is_sql_auto_is_null_enabled:
  238. # SQL_AUTO_IS_NULL controls whether an AUTO_INCREMENT column on
  239. # a recently inserted row will return when the field is tested
  240. # for NULL. Disabling this brings this aspect of MySQL in line
  241. # with SQL standards.
  242. assignments.append("SET SQL_AUTO_IS_NULL = 0")
  243. if self.isolation_level:
  244. assignments.append(
  245. "SET SESSION TRANSACTION ISOLATION LEVEL %s"
  246. % self.isolation_level.upper()
  247. )
  248. if assignments:
  249. with self.cursor() as cursor:
  250. cursor.execute("; ".join(assignments))
  251. @async_unsafe
  252. def create_cursor(self, name=None):
  253. cursor = self.connection.cursor()
  254. return CursorWrapper(cursor)
  255. def _rollback(self):
  256. try:
  257. BaseDatabaseWrapper._rollback(self)
  258. except Database.NotSupportedError:
  259. pass
  260. def _set_autocommit(self, autocommit):
  261. with self.wrap_database_errors:
  262. self.connection.autocommit(autocommit)
  263. def disable_constraint_checking(self):
  264. """
  265. Disable foreign key checks, primarily for use in adding rows with
  266. forward references. Always return True to indicate constraint checks
  267. need to be re-enabled.
  268. """
  269. with self.cursor() as cursor:
  270. cursor.execute("SET foreign_key_checks=0")
  271. return True
  272. def enable_constraint_checking(self):
  273. """
  274. Re-enable foreign key checks after they have been disabled.
  275. """
  276. # Override needs_rollback in case constraint_checks_disabled is
  277. # nested inside transaction.atomic.
  278. self.needs_rollback, needs_rollback = False, self.needs_rollback
  279. try:
  280. with self.cursor() as cursor:
  281. cursor.execute("SET foreign_key_checks=1")
  282. finally:
  283. self.needs_rollback = needs_rollback
  284. def check_constraints(self, table_names=None):
  285. """
  286. Check each table name in `table_names` for rows with invalid foreign
  287. key references. This method is intended to be used in conjunction with
  288. `disable_constraint_checking()` and `enable_constraint_checking()`, to
  289. determine if rows with invalid references were entered while constraint
  290. checks were off.
  291. """
  292. with self.cursor() as cursor:
  293. if table_names is None:
  294. table_names = self.introspection.table_names(cursor)
  295. for table_name in table_names:
  296. primary_key_column_name = self.introspection.get_primary_key_column(
  297. cursor, table_name
  298. )
  299. if not primary_key_column_name:
  300. continue
  301. relations = self.introspection.get_relations(cursor, table_name)
  302. for column_name, (
  303. referenced_column_name,
  304. referenced_table_name,
  305. ) in relations.items():
  306. cursor.execute(
  307. """
  308. SELECT REFERRING.`%s`, REFERRING.`%s` FROM `%s` as REFERRING
  309. LEFT JOIN `%s` as REFERRED
  310. ON (REFERRING.`%s` = REFERRED.`%s`)
  311. WHERE REFERRING.`%s` IS NOT NULL AND REFERRED.`%s` IS NULL
  312. """
  313. % (
  314. primary_key_column_name,
  315. column_name,
  316. table_name,
  317. referenced_table_name,
  318. column_name,
  319. referenced_column_name,
  320. column_name,
  321. referenced_column_name,
  322. )
  323. )
  324. for bad_row in cursor.fetchall():
  325. raise IntegrityError(
  326. "The row in table '%s' with primary key '%s' has an "
  327. "invalid foreign key: %s.%s contains a value '%s' that "
  328. "does not have a corresponding value in %s.%s."
  329. % (
  330. table_name,
  331. bad_row[0],
  332. table_name,
  333. column_name,
  334. bad_row[1],
  335. referenced_table_name,
  336. referenced_column_name,
  337. )
  338. )
  339. def is_usable(self):
  340. try:
  341. self.connection.ping()
  342. except Database.Error:
  343. return False
  344. else:
  345. return True
  346. @cached_property
  347. def display_name(self):
  348. return "MariaDB" if self.mysql_is_mariadb else "MySQL"
  349. @cached_property
  350. def data_type_check_constraints(self):
  351. if self.features.supports_column_check_constraints:
  352. check_constraints = {
  353. "PositiveBigIntegerField": "`%(column)s` >= 0",
  354. "PositiveIntegerField": "`%(column)s` >= 0",
  355. "PositiveSmallIntegerField": "`%(column)s` >= 0",
  356. }
  357. if self.mysql_is_mariadb and self.mysql_version < (10, 4, 3):
  358. # MariaDB < 10.4.3 doesn't automatically use the JSON_VALID as
  359. # a check constraint.
  360. check_constraints["JSONField"] = "JSON_VALID(`%(column)s`)"
  361. return check_constraints
  362. return {}
  363. @cached_property
  364. def mysql_server_data(self):
  365. with self.temporary_connection() as cursor:
  366. # Select some server variables and test if the time zone
  367. # definitions are installed. CONVERT_TZ returns NULL if 'UTC'
  368. # timezone isn't loaded into the mysql.time_zone table.
  369. cursor.execute(
  370. """
  371. SELECT VERSION(),
  372. @@sql_mode,
  373. @@default_storage_engine,
  374. @@sql_auto_is_null,
  375. @@lower_case_table_names,
  376. CONVERT_TZ('2001-01-01 01:00:00', 'UTC', 'UTC') IS NOT NULL
  377. """
  378. )
  379. row = cursor.fetchone()
  380. return {
  381. "version": row[0],
  382. "sql_mode": row[1],
  383. "default_storage_engine": row[2],
  384. "sql_auto_is_null": bool(row[3]),
  385. "lower_case_table_names": bool(row[4]),
  386. "has_zoneinfo_database": bool(row[5]),
  387. }
  388. @cached_property
  389. def mysql_server_info(self):
  390. return self.mysql_server_data["version"]
  391. @cached_property
  392. def mysql_version(self):
  393. match = server_version_re.match(self.mysql_server_info)
  394. if not match:
  395. raise Exception(
  396. "Unable to determine MySQL version from version string %r"
  397. % self.mysql_server_info
  398. )
  399. return tuple(int(x) for x in match.groups())
  400. @cached_property
  401. def mysql_is_mariadb(self):
  402. return "mariadb" in self.mysql_server_info.lower()
  403. @cached_property
  404. def sql_mode(self):
  405. sql_mode = self.mysql_server_data["sql_mode"]
  406. return set(sql_mode.split(",") if sql_mode else ())