From 009d2857c5dbe9283086848878c6969d4a73e43b Mon Sep 17 00:00:00 2001 From: Owais Lone Date: Sun, 21 Feb 2016 13:58:15 +0530 Subject: [PATCH] Experimental new loader New class based loader implementation * Potentially Breaking Changes This commit introduces a new `CACHE` setting which when set to true makes the loader cache the contents of the stats files in memory. This means if set to True, the server will have to be restarted every time the stats file contents change or it'll keep serving old, cached URLs. `CACHE` defaults to `not DEBUG` by default. --- README.md | 13 +++ setup.py | 22 ++++- tests/app/settings.py | 2 + tests/app/tests/test_webpack.py | 44 +++++---- tests/tox.ini | 2 +- webpack_loader/__init__.py | 3 + webpack_loader/apps.py | 3 +- webpack_loader/config.py | 32 +++++++ webpack_loader/exceptions.py | 9 ++ webpack_loader/loader.py | 79 ++++++++++++++++ webpack_loader/templatetags/webpack_loader.py | 17 ++-- webpack_loader/utils.py | 94 ++----------------- 12 files changed, 200 insertions(+), 120 deletions(-) create mode 100644 webpack_loader/config.py create mode 100644 webpack_loader/exceptions.py create mode 100644 webpack_loader/loader.py diff --git a/README.md b/README.md index a237179..bdb93e2 100644 --- a/README.md +++ b/README.md @@ -79,6 +79,7 @@ module.exports = { ```python WEBPACK_LOADER = { 'DEFAULT': { + 'CACHE': not DEBUG, 'BUNDLE_DIR_NAME': 'webpack_bundles/', # must end with slash 'STATS_FILE': 'webpack-stats.json', 'POLL_INTERVAL': 0.1, @@ -89,6 +90,18 @@ WEBPACK_LOADER = {
+#### CACHE +```python +WEBPACK_LOADER = { + 'DEFAULT': { + 'CACHE': not DEBUG + } +} +``` +When `CACHE` is set to True, webpack-loader will read the stats file only once and cache the result. This means web workers need to be restarted in order to pick up any changes made to the stats files. + +
+ #### BUNDLE_DIR_NAME ```python WEBPACK_LOADER = { diff --git a/setup.py b/setup.py index 230dc4b..c1652e7 100644 --- a/setup.py +++ b/setup.py @@ -1,15 +1,31 @@ +import os +import re + from setuptools import setup -version = '0.2.4' + +def rel(*parts): + '''returns the relative path to a file wrt to the current directory''' + return os.path.abspath(os.path.join(os.path.dirname(__file__), *parts)) + +with open(rel('README.md')) as handler: + README = handler.read() + +with open(rel('webpack_loader', '__init__.py')) as handler: + INIT_PY = handler.read() + + +VERSION = re.findall("__version__ = '([^']+)'", INIT_PY)[0] setup( name = 'django-webpack-loader', packages = ['webpack_loader', 'webpack_loader/templatetags', 'webpack_loader/contrib'], - version = version, + version = VERSION, description = 'Transparently use webpack with django', + long_description=README, author = 'Owais Lone', author_email = 'hello@owaislone.org', - download_url = 'https://github.com/owais/django-webpack-loader/tarball/{0}'.format(version), + download_url = 'https://github.com/owais/django-webpack-loader/tarball/{0}'.format(VERSION), url = 'https://github.com/owais/django-webpack-loader', # use the URL to the github repo keywords = ['django', 'webpack', 'assets'], # arbitrary keywords data_files = [("", ["LICENSE"])], diff --git a/tests/app/settings.py b/tests/app/settings.py index f5d743e..9913cf3 100644 --- a/tests/app/settings.py +++ b/tests/app/settings.py @@ -110,10 +110,12 @@ STATICFILES_DIRS = ( WEBPACK_LOADER = { 'DEFAULT': { + 'CACHE': False, 'BUNDLE_DIR_NAME': 'bundles/', 'STATS_FILE': os.path.join(BASE_DIR, 'webpack-stats.json'), }, 'APP2': { + 'CACHE': False, 'BUNDLE_DIR_NAME': 'bundles/', 'STATS_FILE': os.path.join(BASE_DIR, 'webpack-stats-app2.json'), } diff --git a/tests/app/tests/test_webpack.py b/tests/app/tests/test_webpack.py index bddcd7e..312557a 100644 --- a/tests/app/tests/test_webpack.py +++ b/tests/app/tests/test_webpack.py @@ -10,21 +10,21 @@ from django.test import RequestFactory, TestCase from django.views.generic.base import TemplateView from django_jinja.builtins import DEFAULT_EXTENSIONS from unittest2 import skipIf -from webpack_loader.utils import (WebpackError, WebpackLoaderBadStatsError, - get_assets, get_bundle, get_config) +from webpack_loader.exceptions import ( + WebpackError, + WebpackLoaderBadStatsError +) +from webpack_loader.utils import get_loader + BUNDLE_PATH = os.path.join(settings.BASE_DIR, 'assets/bundles/') DEFAULT_CONFIG = 'DEFAULT' + class LoaderTestCase(TestCase): def setUp(self): self.factory = RequestFactory() - def clean_dir(self, directory): - if os.path.exists(BUNDLE_PATH): - [os.remove(os.path.join(BUNDLE_PATH, F)) for F in os.listdir(BUNDLE_PATH)] - os.remove(settings.WEBPACK_LOADER[DEFAULT_CONFIG]['STATS_FILE']) - def compile_bundles(self, config, wait=None): if wait: time.sleep(wait) @@ -53,7 +53,7 @@ class LoaderTestCase(TestCase): def test_simple_and_css_extract(self): self.compile_bundles('webpack.config.simple.js') - assets = get_assets(get_config(DEFAULT_CONFIG)) + assets = get_loader(DEFAULT_CONFIG).get_assets() self.assertEqual(assets['status'], 'done') self.assertIn('chunks', assets) @@ -67,13 +67,13 @@ class LoaderTestCase(TestCase): def test_static_url(self): self.compile_bundles('webpack.config.publicPath.js') - assets = get_assets(get_config(DEFAULT_CONFIG)) + assets = get_loader(DEFAULT_CONFIG).get_assets() self.assertEqual(assets['status'], 'done') self.assertEqual(assets['publicPath'], 'http://custom-static-host.com/') def test_code_spliting(self): self.compile_bundles('webpack.config.split.js') - assets = get_assets(get_config(DEFAULT_CONFIG)) + assets = get_loader(DEFAULT_CONFIG).get_assets() self.assertEqual(assets['status'], 'done') self.assertIn('chunks', assets) @@ -149,26 +149,31 @@ class LoaderTestCase(TestCase): #TODO: self.compile_bundles('webpack.config.error.js') try: - get_bundle('main', get_config(DEFAULT_CONFIG)) + get_loader(DEFAULT_CONFIG).get_bundle('main') except WebpackError as e: self.assertIn("Cannot resolve module 'the-library-that-did-not-exist'", str(e)) def test_missing_stats_file(self): - os.remove(settings.WEBPACK_LOADER[DEFAULT_CONFIG]['STATS_FILE']) + stats_file = settings.WEBPACK_LOADER[DEFAULT_CONFIG]['STATS_FILE'] + if os.path.exists(stats_file): + os.remove(stats_file) try: - get_assets(get_config(DEFAULT_CONFIG)) + get_loader(DEFAULT_CONFIG).get_assets() except IOError as e: - expected = 'Error reading {0}. Are you sure webpack has generated the file and the path is correct?'.format(settings.WEBPACK_LOADER[DEFAULT_CONFIG]['STATS_FILE']) + expected = ( + 'Error reading {0}. Are you sure webpack has generated the ' + 'file and the path is correct?' + ).format(stats_file) self.assertIn(expected, str(e)) def test_bad_status_in_production(self): - stats_file = open( + with open( settings.WEBPACK_LOADER[DEFAULT_CONFIG]['STATS_FILE'], 'w' - ) - stats_file.write(json.dumps({'status': 'unexpected-status'})) - stats_file.close() + ) as stats_file: + stats_file.write(json.dumps({'status': 'unexpected-status'})) + try: - get_bundle('main', get_config(DEFAULT_CONFIG)) + get_loader(DEFAULT_CONFIG).get_bundle('main') except WebpackLoaderBadStatsError as e: self.assertIn(( "The stats file does not contain valid data. Make sure " @@ -207,4 +212,3 @@ class LoaderTestCase(TestCase): result.rendered_content elapsed = time.time() - then self.assertTrue(elapsed < wait_for) - diff --git a/tests/tox.ini b/tests/tox.ini index 0ccd2c0..367a76d 100644 --- a/tests/tox.ini +++ b/tests/tox.ini @@ -25,4 +25,4 @@ deps = django18: django>=1.8.0,<1.9.0 django19: django>=1.9.0,<1.10.0 commands = - coverage run --source=webpack_loader manage.py test + coverage run --source=webpack_loader manage.py test {posargs} diff --git a/webpack_loader/__init__.py b/webpack_loader/__init__.py index 6b8fc6c..a952178 100644 --- a/webpack_loader/__init__.py +++ b/webpack_loader/__init__.py @@ -1 +1,4 @@ +__author__ = 'Owais Lone' +__version__ = '0.3.0' + default_app_config = 'webpack_loader.apps.WebpackLoaderConfig' diff --git a/webpack_loader/apps.py b/webpack_loader/apps.py index 141bbe0..294e276 100644 --- a/webpack_loader/apps.py +++ b/webpack_loader/apps.py @@ -3,7 +3,8 @@ from django.apps import AppConfig from .errors import BAD_CONFIG_ERROR -def webpack_cfg_check(app_configs, **kwargs): +def webpack_cfg_check(*args, **kwargs): + '''Test if config is compatible or not''' from django.conf import settings check_failed = False diff --git a/webpack_loader/config.py b/webpack_loader/config.py new file mode 100644 index 0000000..3bb26ca --- /dev/null +++ b/webpack_loader/config.py @@ -0,0 +1,32 @@ +import re + +from django.conf import settings + + +__all__ = ('load_config',) + + +DEFAULT_CONFIG = { + 'DEFAULT': { + 'CACHE': not settings.DEBUG, + 'BUNDLE_DIR_NAME': 'webpack_bundles/', + 'STATS_FILE': 'webpack-stats.json', + # FIXME: Explore usage of fsnotify + 'POLL_INTERVAL': 0.1, + 'IGNORE': ['.+\.hot-update.js', '.+\.map'] + } +} + +user_config = getattr(settings, 'WEBPACK_LOADER', DEFAULT_CONFIG) + +user_config = dict( + (name, dict(DEFAULT_CONFIG['DEFAULT'], **cfg)) + for name, cfg in user_config.items() +) + +for entry in user_config.values(): + entry['ignores'] = [re.compile(I) for I in entry['IGNORE']] + + +def load_config(name): + return user_config[name] diff --git a/webpack_loader/exceptions.py b/webpack_loader/exceptions.py new file mode 100644 index 0000000..c7a5165 --- /dev/null +++ b/webpack_loader/exceptions.py @@ -0,0 +1,9 @@ +__all__ = ('WebpackError', 'WebpackLoaderBadStatsError') + + +class WebpackError(Exception): + pass + + +class WebpackLoaderBadStatsError(Exception): + pass diff --git a/webpack_loader/loader.py b/webpack_loader/loader.py new file mode 100644 index 0000000..869613e --- /dev/null +++ b/webpack_loader/loader.py @@ -0,0 +1,79 @@ +import json +import time + +from django.conf import settings +from django.contrib.staticfiles.storage import staticfiles_storage + +from .exceptions import WebpackError, WebpackLoaderBadStatsError +from .config import load_config + + +class WebpackLoader(object): + _assets = {} + + def __init__(self, name='DEFAULT'): + self.name = name + self.config = load_config(self.name) + + def _load_assets(self): + try: + with open(self.config['STATS_FILE']) as f: + return json.load(f) + except IOError: + raise IOError( + 'Error reading {0}. Are you sure webpack has generated ' + 'the file and the path is correct?'.format( + self.config['STATS_FILE'])) + + def get_assets(self): + if self.config['CACHE']: + if self.name not in self._assets: + self._assets[self.name] = self._load_assets() + return self._assets[self.name] + return self._load_assets() + + def filter_chunks(self, chunks): + for chunk in chunks: + ignore = any(regex.match(chunk['name']) + for regex in self.config['ignores']) + if not ignore: + chunk['url'] = self.get_chunk_url(chunk) + yield chunk + + def get_chunk_url(self, chunk): + public_path = chunk.get('publicPath') + if public_path: + return public_path + + relpath = '{0}{1}'.format( + self.config['BUNDLE_DIR_NAME'], chunk['name'] + ) + return staticfiles_storage.url(relpath) + + def get_bundle(self, bundle_name): + assets = self.get_assets() + + if settings.DEBUG: + # poll when debugging and block request until bundle is compiled + # TODO: support timeouts + while assets['status'] == 'compiling': + time.sleep(self.config['POLL_INTERVAL']) + assets = self.get_assets() + + if assets.get('status') == 'done': + chunks = assets['chunks'][bundle_name] + return self.filter_chunks(chunks) + + elif assets.get('status') == 'error': + if 'file' not in assets: + assets['file'] = '' + error = u""" + {error} in {file} + {message} + """.format(**assets) + raise WebpackError(error) + + raise WebpackLoaderBadStatsError( + "The stats file does not contain valid data. Make sure " + "webpack-bundle-tracker plugin is enabled and try to run " + "webpack again.") diff --git a/webpack_loader/templatetags/webpack_loader.py b/webpack_loader/templatetags/webpack_loader.py index eaea065..a964dc0 100644 --- a/webpack_loader/templatetags/webpack_loader.py +++ b/webpack_loader/templatetags/webpack_loader.py @@ -2,13 +2,13 @@ from django import template from django.conf import settings from django.utils.safestring import mark_safe -from ..utils import get_config, get_assets, get_bundle - +from ..utils import get_loader register = template.Library() def filter_by_extension(bundle, extension): + '''Return only files with the given extension''' for chunk in bundle: if chunk['name'].endswith('.{0}'.format(extension)): yield chunk @@ -17,16 +17,19 @@ def filter_by_extension(bundle, extension): def render_as_tags(bundle): tags = [] for chunk in bundle: - url = chunk.get('publicPath') or chunk['url'] if chunk['name'].endswith('.js'): - tags.append(''.format(url)) + tags.append(( + '' + ).format(chunk['url'])) elif chunk['name'].endswith('.css'): - tags.append(''.format(url)) + tags.append(( + '' + ).format(chunk['url'])) return mark_safe('\n'.join(tags)) def _get_bundle(bundle_name, extension, config): - bundle = get_bundle(bundle_name, get_config(config)) + bundle = get_loader(config).get_bundle(bundle_name) if extension: bundle = filter_by_extension(bundle, extension) return bundle @@ -40,7 +43,7 @@ def render_bundle(bundle_name, extension=None, config='DEFAULT'): @register.simple_tag def webpack_static(asset_name, config='DEFAULT'): return "{0}{1}".format( - get_assets(get_config(config)).get( + get_loader(config).get_assets().get( 'publicPath', getattr(settings, 'STATIC_URL') ), asset_name diff --git a/webpack_loader/utils.py b/webpack_loader/utils.py index f9f5608..012a056 100644 --- a/webpack_loader/utils.py +++ b/webpack_loader/utils.py @@ -1,92 +1,10 @@ -import re -import json -import time - -from django.conf import settings -from django.contrib.staticfiles.storage import staticfiles_storage +from .loader import WebpackLoader -__all__ = ('get_assets', 'get_config', 'get_bundle',) +_loaders = {} -DEFAULT_CONFIG = { - 'DEFAULT': { - 'BUNDLE_DIR_NAME': 'webpack_bundles/', - 'STATS_FILE': 'webpack-stats.json', - # FIXME: Explore usage of fsnotify - 'POLL_INTERVAL': 0.1, - 'IGNORE': ['.+\.hot-update.js', '.+\.map'] - } -} - - -user_config = getattr(settings, 'WEBPACK_LOADER', DEFAULT_CONFIG) - -user_config = dict( - (name, dict(DEFAULT_CONFIG['DEFAULT'], **cfg)) - for name, cfg in user_config.items() -) - -for entry in user_config.values(): - entry['ignores'] = [re.compile(I) for I in entry['IGNORE']] - - -class WebpackError(Exception): - pass - - -class WebpackLoaderBadStatsError(Exception): - pass - - -def get_config(config_name): - return user_config[config_name] - - -def get_assets(config): - try: - with open(config['STATS_FILE']) as f: - return json.load(f) - except IOError: - raise IOError( - 'Error reading {0}. Are you sure webpack has generated the file ' - 'and the path is correct?'.format(config['STATS_FILE'])) - - -def filter_files(files, config): - for F in files: - filename = F['name'] - ignore = any(regex.match(filename) for regex in config['ignores']) - if not ignore: - relpath = '{0}{1}'.format(config['BUNDLE_DIR_NAME'], filename) - F['url'] = staticfiles_storage.url(relpath) - yield F - - -def get_bundle(bundle_name, config): - assets = get_assets(config) - - if settings.DEBUG: - # poll when debugging and block request until bundle is compiled - # TODO: support timeouts - while assets['status'] == 'compiling': - time.sleep(config['POLL_INTERVAL']) - assets = get_assets(config) - - if assets.get('status') == 'done': - files = assets['chunks'][bundle_name] - return filter_files(files, config) - - elif assets.get('status') == 'error': - if 'file' not in assets: - assets['file'] = '' - error = u""" - {error} in {file} - {message} - """.format(**assets) - raise WebpackError(error) - - raise WebpackLoaderBadStatsError( - "The stats file does not contain valid data. Make sure " - "webpack-bundle-tracker plugin is enabled and try to run " - "webpack again.") +def get_loader(config_name): + if config_name not in _loaders: + _loaders[config_name] = WebpackLoader(config_name) + return _loaders[config_name]