# -*- coding: utf-8 -*- import cgi import json import os.path import re from io import BytesIO from urlparse import urlparse from flask import request, Response, send_from_directory, url_for from flask_restful import abort from jinja2 import Environment, PackageLoader from lxml import etree from sh import Command, ErrorReturnCode from werkzeug.exceptions import NotFound from distribution import app from distribution.adapters import serializers from distribution.adapters.clients import amp as amp_client from distribution.models.feed import FeedItem from distribution.utils import string_to_utc_date from distribution.adapters.rules import AMPRuleManager class Adapter(object): """ Adapter base class, serializes and renders data. """ encoding = "UTF-8" mimetype = None content_type = None serializer_class = serializers.FeedSerializer wrap_element = 'items' context = None is_public = False def __init__(self, mimetype=None, encoding=None, content_type=None, context=None): """ Adapter's constructor. :param mimetype: the mimetype to use (as part of the content-type). :param encoding: an optional encoding to add to the content-type. :param content_type: sets a specific content-type in the response; Defaults to the mimetype and encoding params. :param context: context to the adapter (updates the class context). """ if mimetype is not None: self.mimetype = mimetype if encoding is not None: self.encoding = encoding if content_type or not self.content_type: self.build_content_type(content_type) # Override class context self.context = dict(self.context or {}, **(context or {})) # Instantiate serializer self.serializer = self.get_serializer() def build_content_type(self, value): if self.mimetype: self.content_type = "{mimetype}; charset={encoding}" \ .format(mimetype=self.mimetype, encoding=self.encoding) def get_serializer(self): """ Returns a serializer instance to be used on every request. """ return self.serializer_class() def get_context(self, items, **kwargs): creation_date = items[0].data['meta']['created_at'] if items else None parsed = urlparse(request.url) scheme = request.headers.get(app.config['HTTP_SCHEME_HEADER'], 'http') kwargs.update(host=cgi.escape(parsed._replace(scheme=scheme).geturl()), last_build_date=string_to_utc_date(creation_date)) return dict(self.context, **kwargs) @staticmethod def render_content(context): """ Renders the content. The default implementation does nothing (delegates to Flask). :param context: content data including the output from the serializer. :return: a unicode, stream or anything respond() can handle. """ return context def pre_render(self, items): """ Executes pre render process after save. :param items: FeedItem list. """ pass def delete(self, keys): """ Delete pre rendered templates :param keys: FeedItem keys list. """ pass def render(self, items, fields=None, **kwargs): """ This method orchestrates rendering: 1. Calls get_context() to retrieve an initial context, possibly from the given arguments; 2. Serializes the items and includes them in the context, under the key specified by self.wrap_element; 3. Calls render_content(); 4. Calls respond() and returns the result. If wrap_element is None, the context will not be resolved and the serialized items will be passed to render_context directly. :param items: raw data from the object manager (e.g.: search results). :param kwargs: optional extra context. """ if self.wrap_element: context = self.get_context(items, **kwargs) context[self.wrap_element] = self.serializer.serialize(items, fields) else: context = self.serializer.serialize(items, fields) rendered = self.render_content(context) return self.respond(rendered) def respond(self, content): """ Builds and returns a response object from the given content. :param content: data as returned by render_content. :return: an HTTP Response object """ return Response(response=content, status=200, content_type=self.content_type) class JinjaAdapterMixin(object): template_name = None _env = Environment( loader=PackageLoader('distribution', 'templates'), trim_blocks=True, lstrip_blocks=True) def __init__(self, template_name=None, **options): if template_name is not None: self.template_name = template_name super(JinjaAdapterMixin, self).__init__(**options) @property def template(self): """ Looks up a file by its name and returns a Jinja template object. The object is stored in the instance for fast reuse. """ try: return self._template except AttributeError: self._template = self._env.get_template(self.template_name) return self._template def render_template(self, *args, **kwargs): """ Calls the template's render method with the given arguments. """ return self.template.render(*args, **kwargs) def render_content(self, context): """ Renders the template, passing the given content as context. :param content: data content as returned by the serializer. :return: whatever Jinja returns. """ return self.render_template(context) class XMLRenderMixin(object): mimetype = "text/xml" encoding = 'utf-8' def __init__(self, **options): super(XMLRenderMixin, self).__init__(**options) self.xmlparser = etree.XMLParser(strip_cdata=False, encoding=self.encoding) def respond(self, content): encoding = self.encoding try: stream = BytesIO(content.encode(encoding)) xml_content = etree.parse(stream, self.xmlparser) response = etree.tostring(xml_content, xml_declaration=True, encoding=encoding) except etree.XMLSyntaxError: if not app.config['DEBUG']: abort(503, message='Service currently unavailable') # This re-raises the error exposing useful details for debugging response = content.encode(encoding) except (etree.DocumentInvalid, IOError) as e: app.logger.exception(e) abort(500, message='Internal server error') return Response(response=response, status=200, content_type=self.content_type) class SiteMapIndexAdapter(XMLRenderMixin, JinjaAdapterMixin, Adapter): template_name = "sitemap_index.xml" is_public = True def render(self, content, **kwargs): sitemap_pages = self._build_pages(content) rendered_content = self.render_content({'pages': sitemap_pages}) return self.respond(rendered_content) @staticmethod def _build_pages(content): pages = [] for page in xrange(1, content['total_pages']+1): pages.append( url_for('sitemappage', _external=True, _scheme=request.headers.get(app.config['HTTP_SCHEME_HEADER'], 'http'), **{'domain': content['domain'], 'page': page})) return pages class SiteMapAdapter(XMLRenderMixin, JinjaAdapterMixin, Adapter): template_name = "sitemap_page.xml" is_public = True def get_serializer(self): return self.serializer_class(item_serializer=serializers.SitemapURLSerializer) class RSSOutputAdapter(XMLRenderMixin, JinjaAdapterMixin, Adapter): template_name = "feed.rss" context = app.config['RSS_DEFAULT_VALUES'] class JSONOutputAdapter(Adapter): serializer_class = serializers.JSONFeedItemSerializer wrap_element = None def respond(self, content): return content class TemplatedJSONOutputAdapter(JinjaAdapterMixin, Adapter): template_name = 'feed.json' mimetype = "application/json" class JSONAPIOutputAdapter(TemplatedJSONOutputAdapter): template_name = "feed.jsonapi.json" content_type = "application/vnd.api+json" serializer_class = serializers.JSONAPISerializer def render_content(self, context): # consume and store items: items = list(context[self.wrap_element]) context[self.wrap_element] = items context['included'] = self.get_included_data(items) return super(JSONAPIOutputAdapter, self).render_content(context) @staticmethod def get_included_data(primary_data): objects = {} for item in primary_data: for related in item.included: objects.setdefault((related.content_type, related.id), related) return [v for (k, v) in sorted(objects.iteritems())] class AMPAdapter(Adapter): mimetype = 'text/html' is_public = True output_folder = app.config['AMP_OUTPUT_FOLDER'] item_endpoint = app.config['AMP_ITEM_ENDPOINT'] amp_api_user = app.config['AMP_API_USER'] amp_api_key = app.config['AMP_API_KEY'] amp_rule_manager = AMPRuleManager() _amp_current_version = None CRITICAL_ERROR_LIMIT = 10 @property def amp_current_version(self): if self._amp_current_version is None: self._amp_current_version = self.get_amp_version_from_npm() return self._amp_current_version @staticmethod def get_amp_version_from_npm(): """ :return: The current AMP version that exists as output of the npm command """ npm_command = Command("npm") try: json_command = npm_command("list", "@natgeo/modules-amp", "-j", _cwd=app.config['APP_ROOT']) dict_content = json.loads(json_command.stdout) version = dict_content.get('dependencies').get('@natgeo/modules-amp').get('version') except ErrorReturnCode as e: error_content = json.loads(e.stdout) version = error_content.get('dependencies').get('@natgeo/modules-amp').get('version') if not version: msg = ('AMP Adapter: Error while getting module version: ERROR CODE: {}. ' 'CMD: {}') app.logger.error(msg.format(e.exit_code, e.full_cmd)) raise return version def update_amp_files(self): """ Removes all the feeds that have a different AMP version than the actual one in the templates and creates their new template version using the new AMP version """ old_feeds = [] app.logger.info('Latest version of amp is {}'.format(self.amp_current_version)) pattern = r'(^{}_.*)(\.html$)'.format(self.amp_current_version) for file_name in os.listdir(self.output_folder): if not re.match(pattern, file_name): try: # get the feed item id only in case it's possible and add it # to a list of old feeds old_feeds.append(file_name.split('_')[1].replace('.html', '')) except IndexError: pass os.remove(os.path.join(self.output_folder, file_name)) feed_items_to_upgrade = FeedItem.manager.mget(old_feeds) app.logger.info('feeds to update: {}'.format(old_feeds if len(old_feeds) > 0 else 'None')) self.pre_render(feed_items_to_upgrade, background=False) def pre_render(self, items, background=True): for item in items: if self.evaluate_condition(item): self.execute_command(item, background) def delete(self, keys): for item_key in keys: file_path = os.path.join(self.output_folder, '{}_{}.html'.format(self.amp_current_version, item_key)) try: os.remove(file_path) msg = 'AMP Adapter: Deleted AMP Template: {}' app.logger.info(msg.format(item_key)) except OSError: app.logger.info('AMP Adapter: Error deleting the amp file for article: {}. File' ' does not exist'.format(item_key)) if app.config['AMP_CACHE_ENABLED']: try: item = FeedItem.manager.get(item_key) amp_client.update_cache(item.data['uri']) except NotFound: app.logger.info('AMP Adapter: Error deleting the cache for article: {}. Element' ' does not exist'.format(item_key)) def execute_command(self, item, background=True): exit_command = None cwd = os.path.join(os.path.dirname(app.config['APP_ROOT']), 'node_modules/.bin/modules-amp') modules_amp = Command(cwd) def done(cmd, success, exit_code): if not exit_code and os.path.exists(file_path): msg = 'AMP Adapter: Created AMP Template for article: {}' app.logger.info(msg.format(item.key)) if app.config['AMP_CACHE_ENABLED']: amp_client.update_cache(item.data['uri']) app.logger.info('Cache Updated: {}'.format(msg.format(item.key))) else: msg = ('AMP Adapter: Error creating AMP Template for article: {}. ERROR CODE: {}. ' 'CMD: {}') app.logger.error(msg.format(item.key, exit_code, ' '.join(cmd.cmd))) try: file_path = self.output_folder + '{}_{}.html'.format(self.amp_current_version, item.key) exit_command = modules_amp("render", output=file_path, endpoint=self.item_endpoint.format(item.key), user=self.amp_api_user, p=self.amp_api_key, _bg_exc=False, _bg=background, _done=done) except ErrorReturnCode as e: exit_command = int(e.exit_code) return exit_command def evaluate_condition(self, item): domain = item.get('meta', {}).get('domain') sources = item.get('sources', []) genres = item.get('genres', []) content_type = item.get('content_type') current_rule = self.amp_rule_manager.get_amp_rule(content_type) if current_rule: return current_rule.validate(domain, sources, genres) return False def render(self, item, retries=0): if self.evaluate_condition(item) and retries <= app.config.get('AMP_MAX_RETRIES', 1): try: return send_from_directory(self.output_folder, '{}_{}.html'.format(self.amp_current_version, item.key)) except NotFound: exit_command = self.execute_command(item, background=False) if not exit_command or exit_command > self.CRITICAL_ERROR_LIMIT: return self.render(item, retries + 1) return abort(404)