Merge branch 'master' into cache

README.rst

@@ -52,6 +52,23 @@ Examples
 3YOURMIND is running the linter on `Bitbucket Pipelines`_ on every build getting pushed.
 Checkout the ``examples/`` folder to see how you could integrate the linter in your test suite.
 
+Ignoring migrations
+-------------------
+
+You can also ignore migrations by adding this to your migrations:
+
+.. code-block::
+
+    import django_migration_linter as linter
+    # ...
+
+        operations = [
+            linter.IgnoreMigration(),
+            # ...
+        ]
+    # ...
+
+
 Tests
 -----
 
@@ -61,7 +78,7 @@ Contributing
 ------------
 
 First, thank you very much if you want to contribute to the project.
-Please base your work on the ``dev`` branch and also target this branch in your pull request.
+Please base your work on the ``master`` branch and also target this branch in your pull request.
 
 License
 -------

django_migration_linter/__init__.py

@@ -13,3 +13,4 @@
 # limitations under the License.
 
 from .migration_linter import *  # noqa
+from .operations import *  # noqa

django_migration_linter/migration_linter.py

@@ -100,6 +100,12 @@ class MigrationLinter(object):
         analysis_result = analyse_sql_statements(
             sql_statements)
         errors = analysis_result['errors']
+
+        if analysis_result['ignored']:
+            print('IGNORE')
+            self.nb_ignored += 1
+            return
+
         if not errors:
             print('OK')
             self.new_cache[md5hash] = {'result': 'OK'}
@@ -206,7 +212,7 @@ class MigrationLinter(object):
                 utils.clean_bytes_to_str, diff_process.stdout.readlines()):
             # Only gather lines that include added migrations
             if re.search(
-                    '\/{0}\/.*\.py'.format(MIGRATION_FOLDER_NAME),
+                    r'\/{0}\/.*\.py'.format(MIGRATION_FOLDER_NAME),
                     line) and \
                         '__init__' not in line:
                 app_name, migration_name = split_migration_path(line)
@@ -226,7 +232,7 @@ class MigrationLinter(object):
     def _gather_all_migrations(self):
         migrations = []
         for root, dirs, files in os.walk(self.django_path):
-            for file_name in files:
+            for file_name in sorted(files):
                 if os.path.basename(root) == MIGRATION_FOLDER_NAME and \
                         file_name.endswith('.py') and \
                         file_name != '__init__.py':

django_migration_linter/operations.py

@@ -0,0 +1,38 @@
+# Copyright 2018 3YOURMIND GmbH
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from django.db.migrations.operations.base import Operation
+
+IGNORE_MIGRATION_SQL = "-- dml ignores this migration"
+
+
+class IgnoreMigration(Operation):
+
+    reversible = True
+    # Must be true to be generated through the sqlmigrate command
+    reduces_to_sql = True
+
+    def state_forwards(self, app_label, state):
+        pass
+
+    def database_forwards(
+            self, app_label, schema_editor, from_state, to_state):
+        schema_editor.execute(IGNORE_MIGRATION_SQL)
+
+    def database_backwards(
+            self, app_label, schema_editor, from_state, to_state):
+        schema_editor.execute(IGNORE_MIGRATION_SQL + "(reversed)")
+
+    def describe(self):
+        return "The Django migration linter will ignore this migration"

django_migration_linter/sql_analyser.py

@@ -15,6 +15,10 @@
 import re
 import logging
 
+from django_migration_linter.operations import IGNORE_MIGRATION_SQL
+
+IGNORED_MIGRATION = "IGNORED_MIGRATION"
+
 logger = logging.getLogger(__name__)
 
 
@@ -63,29 +67,45 @@ migration_tests = (
         'code': '',
         'fn': has_default,
         'err_msg': ''
+    }, {
+        'code': IGNORED_MIGRATION,
+        'fn': lambda sql, **kw: re.search(IGNORE_MIGRATION_SQL, sql),
+        'err_msg': '',
     }
 )
 
 
 def analyse_sql_statements(sql_statements):
     errors = []
+    ignored = False
     for statement in sql_statements:
         for test in migration_tests:
             if test['fn'](statement, errors=errors):
-                logger.info('Testing {0} -- ERROR'.format(statement))
+                if test['code'] == IGNORED_MIGRATION:
-                table_search = re.search(
+                    logger.info(
-                    'TABLE `([^`]*)`', statement, re.IGNORECASE)
+                        'Testing {0} -- IGNORING MIGRATION'.format(statement)
-                col_search = re.search(
+                    )
-                    'COLUMN `([^`]*)`', statement, re.IGNORECASE)
+                    ignored = True
-                err = {
+                else:
-                    'err_msg': test['err_msg'],
+                    logger.info('Testing {0} -- ERROR'.format(statement))
-                    'code': test['code'],
+                    table_search = re.search(
-                    'table': table_search.group(1) if table_search else None,
+                        'TABLE `([^`]*)`', statement, re.IGNORECASE)
-                    'column': col_search.group(1) if col_search else None
+                    col_search = re.search(
-                }
+                        'COLUMN `([^`]*)`', statement, re.IGNORECASE)
-                errors.append(err)
+                    err = {
+                        'err_msg':
+                            test['err_msg'],
+                        'code':
+                            test['code'],
+                        'table':
+                            table_search.group(1) if table_search else None,
+                        'column':
+                            col_search.group(1) if col_search else None
+                    }
+                    errors.append(err)
             else:
                 logger.info('Testing {0} -- PASSED'.format(statement))
     return {
         'errors': errors,
+        'ignored': ignored
     }

tests/fixtures.py

@@ -40,6 +40,8 @@ CORRECT_PROJECT = os.path.join(
     _FIXTURES_FOLDER, 'test_correct_project/')
 DELETED_MIGRATION_PROJECT = os.path.join(
     _FIXTURES_FOLDER, 'test_deleted_migration_project/')
+IGNORE_MIGRATION_PROJECT = os.path.join(
+    _FIXTURES_FOLDER, 'test_project_ignore_migration/')
 
 NOT_DJANGO_GIT_PROJECT = os.path.join(
     _FIXTURES_FOLDER, 'test_git_project/')

tests/functional/test_cmd_line_call.py

@@ -37,9 +37,14 @@ class CallLinterFromCommandLineTest(unittest.TestCase):
         self.assertEqual(process.returncode, 0)
         lines = list(map(utils.clean_bytes_to_str, process.stdout.readlines()))
         self.assertEqual(len(lines), 5)
-        self.assertTrue(lines[0].endswith('OK'))
+        self.assertEqual(
-        self.assertTrue(lines[1].endswith('OK'))
+            sorted(lines[:3]),
-        self.assertTrue(lines[2].endswith('OK'))
+            sorted([
+                 "(test_app1, 0001_initial)... OK",
+                 "(test_app1, 0002_a_new_null_field)... OK",
+                 "(test_app2, 0001_foo)... OK",
+             ])
+        )
 
     def test_call_linter_cmd_line_errors(self):
         cmd = '{0} --no-cache {1}'.format(
@@ -52,7 +57,6 @@ class CallLinterFromCommandLineTest(unittest.TestCase):
         self.assertNotEqual(process.returncode, 0)
         lines = list(map(utils.clean_bytes_to_str, process.stdout.readlines()))
         self.assertEqual(len(lines), 5)
-        print(lines)
         self.assertTrue(lines[0].endswith('OK'))
         self.assertTrue(lines[1].endswith('ERR'))
         self.assertTrue('RENAMING tables' in lines[2])
@@ -68,9 +72,14 @@ class CallLinterFromCommandLineTest(unittest.TestCase):
         self.assertEqual(process.returncode, 0)
         lines = list(map(utils.clean_bytes_to_str, process.stdout.readlines()))
         self.assertEqual(len(lines), 5)
-        self.assertTrue(lines[0].endswith('OK'))
+        self.assertEqual(
-        self.assertTrue(lines[1].endswith('OK'))
+            sorted(lines[:3]),
-        self.assertTrue(lines[2].endswith('IGNORE'))
+            sorted([
+                 "(test_app1, 0001_initial)... OK",
+                 "(test_app1, 0002_a_new_null_field)... OK",
+                 "(test_app2, 0001_foo)... IGNORE",
+             ])
+        )
 
     def test_call_linter_cmd_line_include_apps(self):
         cmd = '{0} --no-cache {1} --include-apps test_app2'.format(
@@ -83,9 +92,14 @@ class CallLinterFromCommandLineTest(unittest.TestCase):
         self.assertEqual(process.returncode, 0)
         lines = list(map(utils.clean_bytes_to_str, process.stdout.readlines()))
         self.assertEqual(len(lines), 5)
-        self.assertTrue(lines[0].endswith('IGNORE'))
+        self.assertEqual(
-        self.assertTrue(lines[1].endswith('IGNORE'))
+            sorted(lines[:3]),
-        self.assertTrue(lines[2].endswith('OK'))
+            sorted([
+                 "(test_app1, 0001_initial)... IGNORE",
+                 "(test_app1, 0002_a_new_null_field)... IGNORE",
+                 "(test_app2, 0001_foo)... OK",
+             ])
+        )
 
     def test_call_linter_cmd_line_ignore_name(self):
         cmd = '{0} --no-cache {1} --ignore-name 0001_initial'.format(
@@ -98,9 +112,14 @@ class CallLinterFromCommandLineTest(unittest.TestCase):
         self.assertEqual(process.returncode, 0)
         lines = list(map(utils.clean_bytes_to_str, process.stdout.readlines()))
         self.assertEqual(len(lines), 5)
-        self.assertTrue(lines[0].endswith('IGNORE'))
+        self.assertEqual(
-        self.assertTrue(lines[1].endswith('OK'))
+            sorted(lines[:3]),
-        self.assertTrue(lines[2].endswith('OK'))
+            sorted([
+                 "(test_app1, 0001_initial)... IGNORE",
+                 "(test_app1, 0002_a_new_null_field)... OK",
+                 "(test_app2, 0001_foo)... OK",
+             ])
+        )
 
     def test_call_linter_cmd_line_ignore_name_contains(self):
         cmd = '{0} --no-cache {1} --ignore-name-contains 0001'.format(
@@ -113,9 +132,14 @@ class CallLinterFromCommandLineTest(unittest.TestCase):
         self.assertEqual(process.returncode, 0)
         lines = list(map(utils.clean_bytes_to_str, process.stdout.readlines()))
         self.assertEqual(len(lines), 5)
-        self.assertTrue(lines[0].endswith('IGNORE'))
+        self.assertEqual(
-        self.assertTrue(lines[1].endswith('OK'))
+            sorted(lines[:3]),
-        self.assertTrue(lines[2].endswith('IGNORE'))
+            sorted([
+                 "(test_app1, 0001_initial)... IGNORE",
+                 "(test_app1, 0002_a_new_null_field)... OK",
+                 "(test_app2, 0001_foo)... IGNORE",
+             ])
+        )
 
     def test_call_linter_cmd_line_cache(self):
         cache_file = os.path.join(DEFAULT_CACHE_PATH, 'test_correct_project.pickle')

tests/functional/test_migration_linter.py

@@ -56,6 +56,10 @@ class BackwardcompatibilityDetectionTest(unittest.TestCase):
         test_project_path = fixtures.RENAME_TABLE_PROJECT
         self._test_linter_finds_errors(test_project_path)
 
+    def test_ignore_migration(self):
+        test_project_path = fixtures.IGNORE_MIGRATION_PROJECT
+        self._test_linter_finds_no_errors(test_project_path)
+
     def test_accept_not_null_column_followed_by_adding_default(self):
         test_project_path = \
             fixtures.ADD_NOT_NULL_COLUMN_FOLLOWED_BY_DEFAULT_PROJECT

tests/test_project_fixtures/test_project_ignore_migration/.gitignore

@@ -0,0 +1,3 @@
+*.pyc
+__pycache__/
+*sqlite3

tests/test_project_fixtures/test_project_ignore_migration/manage.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+import os
+import sys
+
+if __name__ == "__main__":
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "test_project_ignore_migration.settings")
+    try:
+        from django.core.management import execute_from_command_line
+    except ImportError:
+        # The above import may fail for some other reason. Ensure that the
+        # issue is really that Django is missing to avoid masking other
+        # exceptions on Python 2.
+        try:
+            import django
+        except ImportError:
+            raise ImportError(
+                "Couldn't import Django. Are you sure it's installed and "
+                "available on your PYTHONPATH environment variable? Did you "
+                "forget to activate a virtual environment?"
+            )
+        raise
+    execute_from_command_line(sys.argv)

tests/test_project_fixtures/test_project_ignore_migration/test_app/admin.py

@@ -0,0 +1,3 @@
+from django.contrib import admin
+
+# Register your models here.

tests/test_project_fixtures/test_project_ignore_migration/test_app/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class TestAppConfig(AppConfig):
+    name = 'test_app'

tests/test_project_fixtures/test_project_ignore_migration/test_app/migrations/0001_create_table.py

@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.6 on 2017-10-25 14:15
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='A',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('null_field', models.IntegerField(null=True)),
+            ],
+        ),
+    ]

tests/test_project_fixtures/test_project_ignore_migration/test_app/migrations/0002_rename_table.py

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.6 on 2017-10-25 14:15
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+import django_migration_linter as linter
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('test_app', '0001_create_table'),
+    ]
+
+    operations = [
+        linter.IgnoreMigration(),
+        migrations.RenameModel(
+            old_name='A',
+            new_name='B',
+        ),
+    ]

tests/test_project_fixtures/test_project_ignore_migration/test_app/models.py

@@ -0,0 +1,4 @@
+from django.db import models
+
+class B(models.Model):
+	null_field = models.IntegerField(null=True)

tests/test_project_fixtures/test_project_ignore_migration/test_app/tests.py

@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.

tests/test_project_fixtures/test_project_ignore_migration/test_app/views.py

@@ -0,0 +1,3 @@
+from django.shortcuts import render
+
+# Create your views here.

tests/test_project_fixtures/test_project_ignore_migration/test_project_ignore_migration/settings.py

@@ -0,0 +1,121 @@
+"""
+Django settings for test_project_ignore_migration project.
+
+Generated by 'django-admin startproject' using Django 1.11.6.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/1.11/topics/settings/
+
+For the full list of settings and their values, see
+https://docs.djangoproject.com/en/1.11/ref/settings/
+"""
+
+import os
+
+# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+
+# Quick-start development settings - unsuitable for production
+# See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/
+
+# SECURITY WARNING: keep the secret key used in production secret!
+SECRET_KEY = 'gtb+mohs%gf0+#fr89vw7e&#edmv=q8!iv+n&j_soawk$tkj3^'
+
+# SECURITY WARNING: don't run with debug turned on in production!
+DEBUG = True
+
+ALLOWED_HOSTS = []
+
+
+# Application definition
+
+INSTALLED_APPS = [
+    'django.contrib.admin',
+    'django.contrib.auth',
+    'django.contrib.contenttypes',
+    'django.contrib.sessions',
+    'django.contrib.messages',
+    'django.contrib.staticfiles',
+    'test_app'
+]
+
+MIDDLEWARE = [
+    'django.middleware.security.SecurityMiddleware',
+    'django.contrib.sessions.middleware.SessionMiddleware',
+    'django.middleware.common.CommonMiddleware',
+    'django.middleware.csrf.CsrfViewMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    'django.contrib.messages.middleware.MessageMiddleware',
+    'django.middleware.clickjacking.XFrameOptionsMiddleware',
+]
+
+ROOT_URLCONF = 'test_project_ignore_migration.urls'
+
+TEMPLATES = [
+    {
+        'BACKEND': 'django.template.backends.django.DjangoTemplates',
+        'DIRS': [],
+        'APP_DIRS': True,
+        'OPTIONS': {
+            'context_processors': [
+                'django.template.context_processors.debug',
+                'django.template.context_processors.request',
+                'django.contrib.auth.context_processors.auth',
+                'django.contrib.messages.context_processors.messages',
+            ],
+        },
+    },
+]
+
+WSGI_APPLICATION = 'test_project_ignore_migration.wsgi.application'
+
+
+# Database
+# https://docs.djangoproject.com/en/1.11/ref/settings/#databases
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.sqlite3',
+        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
+    }
+}
+
+
+# Password validation
+# https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators
+
+AUTH_PASSWORD_VALIDATORS = [
+    {
+        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
+    },
+]
+
+
+# Internationalization
+# https://docs.djangoproject.com/en/1.11/topics/i18n/
+
+LANGUAGE_CODE = 'en-us'
+
+TIME_ZONE = 'UTC'
+
+USE_I18N = True
+
+USE_L10N = True
+
+USE_TZ = True
+
+
+# Static files (CSS, JavaScript, Images)
+# https://docs.djangoproject.com/en/1.11/howto/static-files/
+
+STATIC_URL = '/static/'

tests/test_project_fixtures/test_project_ignore_migration/test_project_ignore_migration/urls.py

@@ -0,0 +1,21 @@
+"""linter_test_project URL Configuration
+
+The `urlpatterns` list routes URLs to views. For more information please see:
+    https://docs.djangoproject.com/en/1.11/topics/http/urls/
+Examples:
+Function views
+    1. Add an import:  from my_app import views
+    2. Add a URL to urlpatterns:  url(r'^$', views.home, name='home')
+Class-based views
+    1. Add an import:  from other_app.views import Home
+    2. Add a URL to urlpatterns:  url(r'^$', Home.as_view(), name='home')
+Including another URLconf
+    1. Import the include() function: from django.conf.urls import url, include
+    2. Add a URL to urlpatterns:  url(r'^blog/', include('blog.urls'))
+"""
+from django.conf.urls import url
+from django.contrib import admin
+
+urlpatterns = [
+    url(r'^admin/', admin.site.urls),
+]

tests/test_project_fixtures/test_project_ignore_migration/test_project_ignore_migration/wsgi.py

@@ -0,0 +1,16 @@
+"""
+WSGI config for linter_test_project project.
+
+It exposes the WSGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/1.11/howto/deployment/wsgi/
+"""
+
+import os
+
+from django.core.wsgi import get_wsgi_application
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "test_project_ignore_migration.settings")
+
+application = get_wsgi_application()

tests/unit/test_linter.py

@@ -81,9 +81,11 @@ class TestLinterFunctions(unittest.TestCase):
         linter = MigrationLinter(fixtures.CORRECT_PROJECT)
         migrations = linter._gather_all_migrations()
         self.assertEqual(len(migrations), 3)
-        self.assertEqual(migrations[0][0], 'test_app1')
+        self.assertEqual(
-        self.assertEqual(migrations[0][1], '0001_initial')
+            sorted(migrations),
-        self.assertEqual(migrations[1][0], 'test_app1')
+            sorted([
-        self.assertEqual(migrations[1][1], '0002_a_new_null_field')
+                ("test_app1", "0001_initial"),
-        self.assertEqual(migrations[2][0], 'test_app2')
+                ("test_app1", "0002_a_new_null_field"),
-        self.assertEqual(migrations[2][1], '0001_foo')
+                ("test_app2", "0001_foo"),
+            ])
+        )