This isn't a bug or feature request but rather a discussion to understand the best way to integrate procrastinate
within a Django project, in particular, at the SQL level, as Django as its own migration system.
My first attempt was to use raw SQL and execute each sql/migrations/delta*
delta file, like this:
# Generated by Django 3.0.8 on 2020-07-18 08:39
import os
from django.db import connection
from django.db import migrations
from django.db import transaction
def apply_sql(apps, schema_editor):
import procrastinate
sql_path = os.path.join(
os.path.dirname(procrastinate.__file__),
'sql',
'migrations',
)
migrations = [
"baseline-0.5.0.sql",
"delta_0.5.0_001_drop_started_at_column.sql",
"delta_0.5.0_002_drop_started_at_column.sql",
"delta_0.5.0_003_drop_procrastinate_version_table.sql",
"delta_0.6.0_001_fix_procrastinate_fetch_job.sql",
"delta_0.7.1_001_fix_trigger_status_events_insert.sql",
"delta_0.8.1_001_add_queueing_lock_column.sql",
"delta_0.10.0_001_close_fetch_job_race_condition.sql",
"delta_0.10.0_002_add_defer_job_function.sql",
"delta_0.11.0_003_add_procrastinate_periodic_defers.sql",
"delta_0.12.0_001_add_foreign_key_index.sql",
]
with connection.cursor() as cursor:
with transaction.atomic():
for name in migrations:
full_path = os.path.join(sql_path, name)
with open(full_path, 'rb') as f:
print('Applying {}'.format(full_path))
sql = f.read().decode()
cursor.execute(sql)
def rewind(*args, **kwargs):
pass
class Migration(migrations.Migration):
dependencies = [
('common', '0008_auto_20200701_1317'),
]
operations = [
migrations.RunPython(apply_sql, rewind)
]
However, this crashes, with the following output:
python manage.py migrate common 0009
Operations to perform:
Target specific migration: 0009_procrastinate, from common
Running migrations:
Applying common.0009_procrastinate...Applying /venv/lib/python3.7/site-packages/procrastinate/sql/migrations/baseline-0.5.0.sql
Applying /venv/lib/python3.7/site-packages/procrastinate/sql/migrations/delta_0.5.0_001_drop_started_at_column.sql
Applying /venv/lib/python3.7/site-packages/procrastinate/sql/migrations/delta_0.5.0_002_drop_started_at_column.sql
Applying /venv/lib/python3.7/site-packages/procrastinate/sql/migrations/delta_0.5.0_003_drop_procrastinate_version_table.sql
Applying /venv/lib/python3.7/site-packages/procrastinate/sql/migrations/delta_0.6.0_001_fix_procrastinate_fetch_job.sql
Applying /venv/lib/python3.7/site-packages/procrastinate/sql/migrations/delta_0.7.1_001_fix_trigger_status_events_insert.sql
Applying /venv/lib/python3.7/site-packages/procrastinate/sql/migrations/delta_0.8.1_001_add_queueing_lock_column.sql
Applying /venv/lib/python3.7/site-packages/procrastinate/sql/migrations/delta_0.10.0_001_close_fetch_job_race_condition.sql
Applying /venv/lib/python3.7/site-packages/procrastinate/sql/migrations/delta_0.10.0_002_add_defer_job_function.sql
Applying /venv/lib/python3.7/site-packages/procrastinate/sql/migrations/delta_0.11.0_003_add_procrastinate_periodic_defers.sql
Traceback (most recent call last):
File "/venv/lib/python3.7/site-packages/django/db/backends/utils.py", line 84, in _execute
return self.cursor.execute(sql)
psycopg2.errors.SyntaxError: syntax error at or near ";"
LINE 9: DROP FUNCTION IF EXISTS procrastinate_defer_job;
^
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "manage.py", line 27, in <module>
execute_from_command_line(sys.argv)
File "/venv/lib/python3.7/site-packages/django/core/management/__init__.py", line 401, in execute_from_command_line
utility.execute()
File "/venv/lib/python3.7/site-packages/django/core/management/__init__.py", line 395, in execute
self.fetch_command(subcommand).run_from_argv(self.argv)
File "/venv/lib/python3.7/site-packages/django/core/management/base.py", line 328, in run_from_argv
self.execute(*args, **cmd_options)
File "/venv/lib/python3.7/site-packages/django/core/management/base.py", line 369, in execute
output = self.handle(*args, **options)
File "/venv/lib/python3.7/site-packages/django/core/management/base.py", line 83, in wrapped
res = handle_func(*args, **kwargs)
File "/venv/lib/python3.7/site-packages/django/core/management/commands/migrate.py", line 233, in handle
fake_initial=fake_initial,
File "/venv/lib/python3.7/site-packages/django/db/migrations/executor.py", line 117, in migrate
state = self._migrate_all_forwards(state, plan, full_plan, fake=fake, fake_initial=fake_initial)
File "/venv/lib/python3.7/site-packages/django/db/migrations/executor.py", line 147, in _migrate_all_forwards
state = self.apply_migration(state, migration, fake=fake, fake_initial=fake_initial)
File "/venv/lib/python3.7/site-packages/django/db/migrations/executor.py", line 245, in apply_migration
state = migration.apply(state, schema_editor)
File "/venv/lib/python3.7/site-packages/django/db/migrations/migration.py", line 124, in apply
operation.database_forwards(self.app_label, schema_editor, old_state, project_state)
File "/venv/lib/python3.7/site-packages/django/db/migrations/operations/special.py", line 190, in database_forwards
self.code(from_state.apps, schema_editor)
File "/app/funkwhale_api/common/migrations/0009_procrastinate.py", line 37, in apply_sql
cursor.execute(sql)
File "/venv/lib/python3.7/site-packages/django/db/backends/utils.py", line 100, in execute
return super().execute(sql, params)
File "/venv/lib/python3.7/site-packages/cacheops/transaction.py", line 93, in execute
result = self._no_monkey.execute(self, sql, params)
File "/venv/lib/python3.7/site-packages/django/db/backends/utils.py", line 68, in execute
return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)
File "/venv/lib/python3.7/site-packages/django/db/backends/utils.py", line 77, in _execute_with_wrappers
return executor(sql, params, many, context)
File "/venv/lib/python3.7/site-packages/django/db/backends/utils.py", line 86, in _execute
return self.cursor.execute(sql, params)
File "/venv/lib/python3.7/site-packages/django/db/utils.py", line 90, in __exit__
raise dj_exc_value.with_traceback(traceback) from exc_value
File "/venv/lib/python3.7/site-packages/django/db/backends/utils.py", line 84, in _execute
return self.cursor.execute(sql)
django.db.utils.ProgrammingError: syntax error at or near ";"
LINE 9: DROP FUNCTION IF EXISTS procrastinate_defer_job;
Execution of delta_0.11.0_003_add_procrastinate_periodic_defers.sql
fails with a syntax error, which I don't understand unfortunately. If you have some ideas, I can definitely try these, and possibly document Django integration when I have a working setup :)
Issue type: Bug 🐞 Issue contains: Some SQL 🐘 Issue type: Feature ⭐️ Issue contains: Some Python 🐍 Issue appropriate for: Occasional contributors 😉