Experimental new loader

New class based loader implementation

* Potentially Breaking Changes

    This commit introduces a new `CACHE` setting which when set to true makes the loader cache the contents of the stats files in memory. This means if set to True, the server will have to be restarted every time the stats file contents change or it'll keep serving old, cached URLs. `CACHE` defaults to `not DEBUG` by default.

README.md

@@ -79,6 +79,7 @@ module.exports = {
 ```python
 WEBPACK_LOADER = {
     'DEFAULT': {
+        'CACHE': not DEBUG,
         'BUNDLE_DIR_NAME': 'webpack_bundles/', # must end with slash
         'STATS_FILE': 'webpack-stats.json',
         'POLL_INTERVAL': 0.1,
@@ -89,6 +90,18 @@ WEBPACK_LOADER = {
 
 <br>
 
+#### CACHE
+```python
+WEBPACK_LOADER = {
+    'DEFAULT': {
+        'CACHE': not DEBUG
+    }
+}
+```
+When `CACHE` is set to True, webpack-loader will read the stats file only once and cache the result. This means web workers need to be restarted in order to pick up any changes made to the stats files.
+
+<br>
+
 #### BUNDLE_DIR_NAME
 ```python
 WEBPACK_LOADER = {

setup.py

@@ -1,15 +1,31 @@
+import os
+import re
+
 from setuptools import setup
 
-version = '0.2.4'
+
+def rel(*parts):
+    '''returns the relative path to a file wrt to the current directory'''
+    return os.path.abspath(os.path.join(os.path.dirname(__file__), *parts))
+
+with open(rel('README.md')) as handler:
+    README = handler.read()
+
+with open(rel('webpack_loader', '__init__.py')) as handler:
+    INIT_PY = handler.read()
+
+
+VERSION = re.findall("__version__ = '([^']+)'", INIT_PY)[0]
 
 setup(
   name = 'django-webpack-loader',
   packages = ['webpack_loader', 'webpack_loader/templatetags', 'webpack_loader/contrib'],
-  version = version,
+  version = VERSION,
   description = 'Transparently use webpack with django',
+  long_description=README,
   author = 'Owais Lone',
   author_email = 'hello@owaislone.org',
-  download_url = 'https://github.com/owais/django-webpack-loader/tarball/{0}'.format(version),
+  download_url = 'https://github.com/owais/django-webpack-loader/tarball/{0}'.format(VERSION),
   url = 'https://github.com/owais/django-webpack-loader', # use the URL to the github repo
   keywords = ['django', 'webpack', 'assets'], # arbitrary keywords
   data_files = [("", ["LICENSE"])],

tests/app/settings.py

@@ -110,10 +110,12 @@ STATICFILES_DIRS = (
 
 WEBPACK_LOADER = {
     'DEFAULT': {
+        'CACHE': False,
         'BUNDLE_DIR_NAME': 'bundles/',
         'STATS_FILE': os.path.join(BASE_DIR, 'webpack-stats.json'),
     },
     'APP2': {
+        'CACHE': False,
         'BUNDLE_DIR_NAME': 'bundles/',
         'STATS_FILE': os.path.join(BASE_DIR, 'webpack-stats-app2.json'),
     }

tests/app/tests/test_webpack.py

@@ -10,21 +10,21 @@ from django.test import RequestFactory, TestCase
 from django.views.generic.base import TemplateView
 from django_jinja.builtins import DEFAULT_EXTENSIONS
 from unittest2 import skipIf
-from webpack_loader.utils import (WebpackError, WebpackLoaderBadStatsError,
+from webpack_loader.exceptions import (
-                                  get_assets, get_bundle, get_config)
+    WebpackError,
+    WebpackLoaderBadStatsError
+)
+from webpack_loader.utils import get_loader
+
 
 BUNDLE_PATH = os.path.join(settings.BASE_DIR, 'assets/bundles/')
 DEFAULT_CONFIG = 'DEFAULT'
 
+
 class LoaderTestCase(TestCase):
     def setUp(self):
         self.factory = RequestFactory()
 
-    def clean_dir(self, directory):
-        if os.path.exists(BUNDLE_PATH):
-            [os.remove(os.path.join(BUNDLE_PATH, F)) for F in os.listdir(BUNDLE_PATH)]
-        os.remove(settings.WEBPACK_LOADER[DEFAULT_CONFIG]['STATS_FILE'])
-
     def compile_bundles(self, config, wait=None):
         if wait:
             time.sleep(wait)
@@ -53,7 +53,7 @@ class LoaderTestCase(TestCase):
 
     def test_simple_and_css_extract(self):
         self.compile_bundles('webpack.config.simple.js')
-        assets = get_assets(get_config(DEFAULT_CONFIG))
+        assets = get_loader(DEFAULT_CONFIG).get_assets()
         self.assertEqual(assets['status'], 'done')
         self.assertIn('chunks', assets)
 
@@ -67,13 +67,13 @@ class LoaderTestCase(TestCase):
 
     def test_static_url(self):
         self.compile_bundles('webpack.config.publicPath.js')
-        assets = get_assets(get_config(DEFAULT_CONFIG))
+        assets = get_loader(DEFAULT_CONFIG).get_assets()
         self.assertEqual(assets['status'], 'done')
         self.assertEqual(assets['publicPath'], 'http://custom-static-host.com/')
 
     def test_code_spliting(self):
         self.compile_bundles('webpack.config.split.js')
-        assets = get_assets(get_config(DEFAULT_CONFIG))
+        assets = get_loader(DEFAULT_CONFIG).get_assets()
         self.assertEqual(assets['status'], 'done')
         self.assertIn('chunks', assets)
 
@@ -149,26 +149,31 @@ class LoaderTestCase(TestCase):
         #TODO:
         self.compile_bundles('webpack.config.error.js')
         try:
-            get_bundle('main', get_config(DEFAULT_CONFIG))
+            get_loader(DEFAULT_CONFIG).get_bundle('main')
         except WebpackError as e:
             self.assertIn("Cannot resolve module 'the-library-that-did-not-exist'", str(e))
 
     def test_missing_stats_file(self):
-        os.remove(settings.WEBPACK_LOADER[DEFAULT_CONFIG]['STATS_FILE'])
+        stats_file = settings.WEBPACK_LOADER[DEFAULT_CONFIG]['STATS_FILE']
+        if os.path.exists(stats_file):
+            os.remove(stats_file)
         try:
-            get_assets(get_config(DEFAULT_CONFIG))
+            get_loader(DEFAULT_CONFIG).get_assets()
         except IOError as e:
-            expected = 'Error reading {0}. Are you sure webpack has generated the file and the path is correct?'.format(settings.WEBPACK_LOADER[DEFAULT_CONFIG]['STATS_FILE'])
+            expected = (
+                'Error reading {0}. Are you sure webpack has generated the '
+                'file and the path is correct?'
+            ).format(stats_file)
             self.assertIn(expected, str(e))
 
     def test_bad_status_in_production(self):
-        stats_file = open(
+        with open(
             settings.WEBPACK_LOADER[DEFAULT_CONFIG]['STATS_FILE'], 'w'
-        )
+        ) as stats_file:
-        stats_file.write(json.dumps({'status': 'unexpected-status'}))
+            stats_file.write(json.dumps({'status': 'unexpected-status'}))
-        stats_file.close()
+
         try:
-            get_bundle('main', get_config(DEFAULT_CONFIG))
+            get_loader(DEFAULT_CONFIG).get_bundle('main')
         except WebpackLoaderBadStatsError as e:
             self.assertIn((
                 "The stats file does not contain valid data. Make sure "
@@ -207,4 +212,3 @@ class LoaderTestCase(TestCase):
             result.rendered_content
             elapsed = time.time() - then
             self.assertTrue(elapsed < wait_for)
-

tests/tox.ini

@@ -25,4 +25,4 @@ deps =
     django18: django>=1.8.0,<1.9.0
     django19: django>=1.9.0,<1.10.0
 commands =
-    coverage run --source=webpack_loader manage.py test
+    coverage run --source=webpack_loader manage.py test {posargs}

webpack_loader/__init__.py

@@ -1 +1,4 @@
+__author__ = 'Owais Lone'
+__version__ = '0.3.0'
+
 default_app_config = 'webpack_loader.apps.WebpackLoaderConfig'

webpack_loader/apps.py

@@ -3,7 +3,8 @@ from django.apps import AppConfig
 from .errors import BAD_CONFIG_ERROR
 
 
-def webpack_cfg_check(app_configs, **kwargs):
+def webpack_cfg_check(*args, **kwargs):
+    '''Test if config is compatible or not'''
     from django.conf import settings
 
     check_failed = False

webpack_loader/config.py

@@ -0,0 +1,32 @@
+import re
+
+from django.conf import settings
+
+
+__all__ = ('load_config',)
+
+
+DEFAULT_CONFIG = {
+    'DEFAULT': {
+        'CACHE': not settings.DEBUG,
+        'BUNDLE_DIR_NAME': 'webpack_bundles/',
+        'STATS_FILE': 'webpack-stats.json',
+        # FIXME: Explore usage of fsnotify
+        'POLL_INTERVAL': 0.1,
+        'IGNORE': ['.+\.hot-update.js', '.+\.map']
+    }
+}
+
+user_config = getattr(settings, 'WEBPACK_LOADER', DEFAULT_CONFIG)
+
+user_config = dict(
+    (name, dict(DEFAULT_CONFIG['DEFAULT'], **cfg))
+    for name, cfg in user_config.items()
+)
+
+for entry in user_config.values():
+    entry['ignores'] = [re.compile(I) for I in entry['IGNORE']]
+
+
+def load_config(name):
+    return user_config[name]

webpack_loader/exceptions.py

@@ -0,0 +1,9 @@
+__all__ = ('WebpackError', 'WebpackLoaderBadStatsError')
+
+
+class WebpackError(Exception):
+    pass
+
+
+class WebpackLoaderBadStatsError(Exception):
+    pass

webpack_loader/loader.py

@@ -0,0 +1,79 @@
+import json
+import time
+
+from django.conf import settings
+from django.contrib.staticfiles.storage import staticfiles_storage
+
+from .exceptions import WebpackError, WebpackLoaderBadStatsError
+from .config import load_config
+
+
+class WebpackLoader(object):
+    _assets = {}
+
+    def __init__(self, name='DEFAULT'):
+        self.name = name
+        self.config = load_config(self.name)
+
+    def _load_assets(self):
+        try:
+            with open(self.config['STATS_FILE']) as f:
+                return json.load(f)
+        except IOError:
+            raise IOError(
+                'Error reading {0}. Are you sure webpack has generated '
+                'the file and the path is correct?'.format(
+                    self.config['STATS_FILE']))
+
+    def get_assets(self):
+        if self.config['CACHE']:
+            if self.name not in self._assets:
+                self._assets[self.name] = self._load_assets()
+            return self._assets[self.name]
+        return self._load_assets()
+
+    def filter_chunks(self, chunks):
+        for chunk in chunks:
+            ignore = any(regex.match(chunk['name'])
+                         for regex in self.config['ignores'])
+            if not ignore:
+                chunk['url'] = self.get_chunk_url(chunk)
+                yield chunk
+
+    def get_chunk_url(self, chunk):
+        public_path = chunk.get('publicPath')
+        if public_path:
+            return public_path
+
+        relpath = '{0}{1}'.format(
+            self.config['BUNDLE_DIR_NAME'], chunk['name']
+        )
+        return staticfiles_storage.url(relpath)
+
+    def get_bundle(self, bundle_name):
+        assets = self.get_assets()
+
+        if settings.DEBUG:
+            # poll when debugging and block request until bundle is compiled
+            # TODO: support timeouts
+            while assets['status'] == 'compiling':
+                time.sleep(self.config['POLL_INTERVAL'])
+                assets = self.get_assets()
+
+        if assets.get('status') == 'done':
+            chunks = assets['chunks'][bundle_name]
+            return self.filter_chunks(chunks)
+
+        elif assets.get('status') == 'error':
+            if 'file' not in assets:
+                assets['file'] = ''
+            error = u"""
+            {error} in {file}
+            {message}
+            """.format(**assets)
+            raise WebpackError(error)
+
+        raise WebpackLoaderBadStatsError(
+            "The stats file does not contain valid data. Make sure "
+            "webpack-bundle-tracker plugin is enabled and try to run "
+            "webpack again.")

webpack_loader/templatetags/webpack_loader.py

@@ -2,13 +2,13 @@ from django import template
 from django.conf import settings
 from django.utils.safestring import mark_safe
 
-from ..utils import get_config, get_assets, get_bundle
+from ..utils import get_loader
-
 
 register = template.Library()
 
 
 def filter_by_extension(bundle, extension):
+    '''Return only files with the given extension'''
     for chunk in bundle:
         if chunk['name'].endswith('.{0}'.format(extension)):
             yield chunk
@@ -17,16 +17,19 @@ def filter_by_extension(bundle, extension):
 def render_as_tags(bundle):
     tags = []
     for chunk in bundle:
-        url = chunk.get('publicPath') or chunk['url']
         if chunk['name'].endswith('.js'):
-            tags.append('<script type="text/javascript" src="{0}"></script>'.format(url))
+            tags.append((
+                '<script type="text/javascript" src="{0}"></script>'
+            ).format(chunk['url']))
         elif chunk['name'].endswith('.css'):
-            tags.append('<link type="text/css" href="{0}" rel="stylesheet"/>'.format(url))
+            tags.append((
+                '<link type="text/css" href="{0}" rel="stylesheet"/>'
+            ).format(chunk['url']))
     return mark_safe('\n'.join(tags))
 
 
 def _get_bundle(bundle_name, extension, config):
-    bundle = get_bundle(bundle_name, get_config(config))
+    bundle = get_loader(config).get_bundle(bundle_name)
     if extension:
         bundle = filter_by_extension(bundle, extension)
     return bundle
@@ -40,7 +43,7 @@ def render_bundle(bundle_name, extension=None, config='DEFAULT'):
 @register.simple_tag
 def webpack_static(asset_name, config='DEFAULT'):
     return "{0}{1}".format(
-        get_assets(get_config(config)).get(
+        get_loader(config).get_assets().get(
             'publicPath', getattr(settings, 'STATIC_URL')
         ),
         asset_name

webpack_loader/utils.py

@@ -1,92 +1,10 @@
-import re
+from .loader import WebpackLoader
-import json
-import time
-
-from django.conf import settings
-from django.contrib.staticfiles.storage import staticfiles_storage
 
 
-__all__ = ('get_assets', 'get_config', 'get_bundle',)
+_loaders = {}
 
 
-DEFAULT_CONFIG = {
+def get_loader(config_name):
-    'DEFAULT': {
+    if config_name not in _loaders:
-        'BUNDLE_DIR_NAME': 'webpack_bundles/',
+        _loaders[config_name] = WebpackLoader(config_name)
-        'STATS_FILE': 'webpack-stats.json',
+    return _loaders[config_name]
-        # FIXME: Explore usage of fsnotify
-        'POLL_INTERVAL': 0.1,
-        'IGNORE': ['.+\.hot-update.js', '.+\.map']
-    }
-}
-
-
-user_config = getattr(settings, 'WEBPACK_LOADER', DEFAULT_CONFIG)
-
-user_config = dict(
-    (name, dict(DEFAULT_CONFIG['DEFAULT'], **cfg))
-    for name, cfg in user_config.items()
-)
-
-for entry in user_config.values():
-    entry['ignores'] = [re.compile(I) for I in entry['IGNORE']]
-
-
-class WebpackError(Exception):
-    pass
-
-
-class WebpackLoaderBadStatsError(Exception):
-    pass
-
-
-def get_config(config_name):
-    return user_config[config_name]
-
-
-def get_assets(config):
-    try:
-        with open(config['STATS_FILE']) as f:
-            return json.load(f)
-    except IOError:
-        raise IOError(
-            'Error reading {0}. Are you sure webpack has generated the file '
-            'and the path is correct?'.format(config['STATS_FILE']))
-
-
-def filter_files(files, config):
-    for F in files:
-        filename = F['name']
-        ignore = any(regex.match(filename) for regex in config['ignores'])
-        if not ignore:
-            relpath = '{0}{1}'.format(config['BUNDLE_DIR_NAME'], filename)
-            F['url'] = staticfiles_storage.url(relpath)
-            yield F
-
-
-def get_bundle(bundle_name, config):
-    assets = get_assets(config)
-
-    if settings.DEBUG:
-        # poll when debugging and block request until bundle is compiled
-        # TODO: support timeouts
-        while assets['status'] == 'compiling':
-            time.sleep(config['POLL_INTERVAL'])
-            assets = get_assets(config)
-
-    if assets.get('status') == 'done':
-        files = assets['chunks'][bundle_name]
-        return filter_files(files, config)
-
-    elif assets.get('status') == 'error':
-        if 'file' not in assets:
-            assets['file'] = ''
-        error = u"""
-        {error} in {file}
-        {message}
-        """.format(**assets)
-        raise WebpackError(error)
-
-    raise WebpackLoaderBadStatsError(
-        "The stats file does not contain valid data. Make sure "
-        "webpack-bundle-tracker plugin is enabled and try to run "
-        "webpack again.")