#!/usr/bin/env python # # Copyright 2007 Google Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # """Handles blob store uploading and serving in the front-end. Includes a WSGI application that handles upload requests and inserts the contents into the blob store. """ import base64 import cgi import cStringIO import datetime import email.generator import email.message from email.mime import multipart import hashlib import logging import os import random import re import sys import time import urlparse import google import webob.exc from google.appengine.api import apiproxy_stub_map from google.appengine.api import datastore from google.appengine.api import datastore_errors from google.appengine.api.blobstore import blobstore from google.appengine.ext.cloudstorage import cloudstorage_stub from google.appengine.tools.devappserver2 import constants # Upload URL path. UPLOAD_URL_PATH = '_ah/upload/' # Upload URL pattern. _UPLOAD_URL_PATTERN = re.compile(r'/%s(.*)' % UPLOAD_URL_PATH) # Pattern for MIME types. _MIME_PATTERN = re.compile(r'([^/; ]+)/([^/; ]+)$') # These are environment variables that do not make any sense to transmit because # the values contained in them would be obsolete after the request has been # transformed from full upload objects to blob-info records. _STRIPPED_ENVIRON = frozenset(('HTTP_CONTENT_LENGTH', 'HTTP_CONTENT_MD5', 'HTTP_CONTENT_TYPE', )) # These are the MIME headers that need to be removed from the external-body # message, because we are going to set our own. # cgi.FieldStorage makes these all lowercase. _STRIPPED_FILE_HEADERS = frozenset(('content-type', 'content-md5', 'content-length', )) # The maximum length of the content-type and filename fields as dictated by # the maximum length of a string in the datastore _MAX_STRING_NAME_LENGTH = 500 class Error(Exception): """Base class for upload processing errors.""" class _InvalidMIMETypeFormatError(Error): """MIME type was formatted incorrectly.""" class _TooManyConflictsError(Error): """There were too many conflicts generating a blob key.""" class _InvalidMetadataError(Error): """The filename or content type of the entity was not a valid UTF-8 string.""" def _get_blob_storage(): """Gets the BlobStorage instance from the API proxy stub map. Returns: The BlobStorage instance as registered with blobstore API in stub map. """ return apiproxy_stub_map.apiproxy.GetStub('blobstore').storage def _generate_blob_key(time_func=time.time, random_func=random.random): """Generate a unique blob key. The key is generated using the current time stamp combined with a random number. The two values are hashed with MD5 and then base-64 encoded (url-safe). The new key is checked to see if it already exists within the datastore and the random number is regenerated until there is no match. Args: time_func: Function that generates a timestamp, as a floating-point number representing seconds since the epoch in UTC. Used for dependency injection (allows for predictable results during tests). random_func: Function used for generating the random number. Used for dependency injection (allows for predictable results during tests). Returns: String version of the blob key that is unique within the __BlobInfo__ datastore. Raises: _TooManyConflictsError: There are too many name conflicts. """ timestamp = str(time_func()) tries = 0 while tries < 10: number = str(random_func()) digester = hashlib.md5() digester.update(timestamp) digester.update(number) blob_key = base64.urlsafe_b64encode(digester.digest()) datastore_key = datastore.Key.from_path(blobstore.BLOB_INFO_KIND, blob_key, namespace='') try: datastore.Get(datastore_key) tries += 1 except datastore_errors.EntityNotFoundError: return blob_key raise _TooManyConflictsError() def _split_mime_type(mime_type): """Split MIME type into main type and subtype. Args: mime_type: The full MIME type string. Returns: (main, sub): main: Main part of MIME type (e.g., application, image, text, etc). sub: Subtype part of MIME type (e.g., pdf, png, html, etc). Raises: _InvalidMIMETypeFormatError: mime_type is incorrectly formatted. """ if mime_type: match = _MIME_PATTERN.match(mime_type) if not match: raise _InvalidMIMETypeFormatError( 'Incorrectly formatted MIME type: %s' % mime_type) return match.groups() else: return 'application', 'octet-stream' class Application(object): """A WSGI middleware application for handling blobstore upload requests. This application will handle all uploaded files in a POST request, store the results in the blob-storage, close the upload session and forward the request on to another WSGI application, with the environment transformed so that the uploaded file contents are replaced with their blob keys. """ def __init__(self, forward_app, get_blob_storage=_get_blob_storage, generate_blob_key=_generate_blob_key, now_func=datetime.datetime.now): """Constructs a new Application. Args: forward_app: A WSGI application to forward successful upload requests to. get_blob_storage: Callable that returns a BlobStorage instance. The default is fine, but may be overridden for testing purposes. generate_blob_key: Function used for generating unique blob keys. now_func: Function that returns the current timestamp. """ self._forward_app = forward_app self._blob_storage = get_blob_storage() self._generate_blob_key = generate_blob_key self._now_func = now_func def abort(self, code, detail=None): """Aborts the application by raising a webob.exc.HTTPException. Args: code: HTTP status code int. detail: Optional detail message str. Raises: webob.exc.HTTPException: Always. """ exception = webob.exc.status_map[code]() if detail: exception.detail = detail raise exception def store_blob(self, content_type, filename, md5_hash, blob_file, creation): """Store a supplied form-data item to the blobstore. The appropriate metadata is stored into the datastore. Args: content_type: The MIME content type of the uploaded file. filename: The filename of the uploaded file. md5_hash: MD5 hash of the file contents, as a hashlib hash object. blob_file: A file-like object containing the contents of the file. creation: datetime.datetime instance to associate with new blobs creation time. This parameter is provided so that all blobs in the same upload form can have the same creation date. Returns: datastore.Entity('__BlobInfo__') associated with the upload. Raises: _TooManyConflictsError if there were too many name conflicts generating a blob key. """ blob_key = self._generate_blob_key() # Store the blob contents in the blobstore. self._blob_storage.StoreBlob(blob_key, blob_file) # Store the blob metadata in the datastore as a __BlobInfo__ entity. blob_entity = datastore.Entity('__BlobInfo__', name=str(blob_key), namespace='') blob_entity['content_type'] = content_type blob_entity['creation'] = creation blob_entity['filename'] = filename blob_entity['md5_hash'] = md5_hash.hexdigest() blob_entity['size'] = blob_file.tell() datastore.Put(blob_entity) return blob_entity def store_gs_file(self, content_type, gs_filename, blob_file, filename): """Store a supplied form-data item to GS. Delegate all the work of gs file creation to CloudStorageStub. Args: content_type: The MIME content type of the uploaded file. gs_filename: The gs filename to create of format bucket/filename. blob_file: A file-like object containing the contents of the file. filename: user provided filename. Returns: datastore.Entity('__GsFileInfo__') associated with the upload. """ gs_stub = cloudstorage_stub.CloudStorageStub(self._blob_storage) blobkey = gs_stub.post_start_creation('/' + gs_filename, {'content-type': content_type}) content = blob_file.read() return gs_stub.put_continue_creation( blobkey, content, (0, len(content) - 1), len(content), filename) def _preprocess_data(self, content_type, blob_file, filename, base64_encoding): """Preprocess data and metadata before storing them. Args: content_type: The MIME content type of the uploaded file. blob_file: A file-like object containing the contents of the file. filename: The filename of the uploaded file. base64_encoding: True, if the file contents are base-64 encoded. Returns: (content_type, blob_file, filename) after proper preprocessing. Raises: _InvalidMetadataError: when metadata are not utf-8 encoded. """ if base64_encoding: blob_file = cStringIO.StringIO(base64.urlsafe_b64decode(blob_file.read())) # If content_type or filename are bytes, assume UTF-8 encoding. try: if not isinstance(content_type, unicode): content_type = content_type.decode('utf-8') if filename and not isinstance(filename, unicode): filename = filename.decode('utf-8') except UnicodeDecodeError: raise _InvalidMetadataError( 'The uploaded entity contained invalid UTF-8 metadata. This may be ' 'because the page containing the upload form was served with a ' 'charset other than "utf-8".') return content_type, blob_file, filename def store_and_build_forward_message(self, form, boundary=None, max_bytes_per_blob=None, max_bytes_total=None, bucket_name=None): """Reads form data, stores blobs data and builds the forward request. This finds all of the file uploads in a set of form fields, converting them into blobs and storing them in the blobstore. It also generates the HTTP request to forward to the user's application. Args: form: cgi.FieldStorage instance representing the whole form derived from original POST data. boundary: The optional boundary to use for the resulting form. If omitted, one is randomly generated. max_bytes_per_blob: The maximum size in bytes that any single blob in the form is allowed to be. max_bytes_total: The maximum size in bytes that the total of all blobs in the form is allowed to be. bucket_name: The name of the Google Storage bucket to store the uploaded files. Returns: A tuple (content_type, content_text), where content_type is the value of the Content-Type header, and content_text is a string containing the body of the HTTP request to forward to the application. Raises: webob.exc.HTTPException: The upload failed. """ message = multipart.MIMEMultipart('form-data', boundary) creation = self._now_func() total_bytes_uploaded = 0 created_blobs = [] mime_type_error = None too_many_conflicts = False upload_too_large = False filename_too_large = False content_type_too_large = False # Extract all of the individual form items out of the FieldStorage. form_items = [] # Sorting of forms is done merely to make testing a little easier since # it means blob-keys are generated in a predictable order. for key in sorted(form): form_item = form[key] if isinstance(form_item, list): form_items.extend(form_item) else: form_items.append(form_item) for form_item in form_items: disposition_parameters = {'name': form_item.name} variable = email.message.Message() if form_item.filename is None: # Copy as is variable.add_header('Content-Type', 'text/plain') variable.set_payload(form_item.value) else: # If there is no filename associated with this field it means that the # file form field was not filled in. This blob should not be created # and forwarded to success handler. if not form_item.filename: continue disposition_parameters['filename'] = form_item.filename try: main_type, sub_type = _split_mime_type(form_item.type) except _InvalidMIMETypeFormatError, ex: mime_type_error = str(ex) break # Seek to the end of file and use the pos as the length. form_item.file.seek(0, os.SEEK_END) content_length = form_item.file.tell() form_item.file.seek(0) total_bytes_uploaded += content_length if max_bytes_per_blob is not None: if content_length > max_bytes_per_blob: upload_too_large = True break if max_bytes_total is not None: if total_bytes_uploaded > max_bytes_total: upload_too_large = True break if form_item.filename is not None: if len(form_item.filename) > _MAX_STRING_NAME_LENGTH: filename_too_large = True break if form_item.type is not None: if len(form_item.type) > _MAX_STRING_NAME_LENGTH: content_type_too_large = True break # Compute the MD5 hash of the upload. digester = hashlib.md5() while True: block = form_item.file.read(1 << 20) if not block: break digester.update(block) form_item.file.seek(0) # Create the external body message containing meta-data about the blob. external = email.message.Message() external.add_header('Content-Type', '%s/%s' % (main_type, sub_type), **form_item.type_options) # NOTE: This is in violation of RFC 2616 (Content-MD5 should be the # base-64 encoding of the binary hash, not the hex digest), but it is # consistent with production. content_md5 = base64.urlsafe_b64encode(digester.hexdigest()) # Create header MIME message headers = dict(form_item.headers) for name in _STRIPPED_FILE_HEADERS: if name in headers: del headers[name] headers['Content-Length'] = str(content_length) headers[blobstore.UPLOAD_INFO_CREATION_HEADER] = ( blobstore._format_creation(creation)) headers['Content-MD5'] = content_md5 gs_filename = None if bucket_name: random_key = str(self._generate_blob_key()) gs_filename = '%s/fake-%s' % (bucket_name, random_key) headers[blobstore.CLOUD_STORAGE_OBJECT_HEADER] = ( blobstore.GS_PREFIX + gs_filename) for key, value in headers.iteritems(): external.add_header(key, value) # Add disposition parameters (a clone of the outer message's field). if not external.get('Content-Disposition'): external.add_header('Content-Disposition', 'form-data', **disposition_parameters) base64_encoding = (form_item.headers.get('Content-Transfer-Encoding') == 'base64') content_type, blob_file, filename = self._preprocess_data( external['content-type'], form_item.file, form_item.filename, base64_encoding) # Store the actual contents to storage. if gs_filename: info_entity = self.store_gs_file( content_type, gs_filename, blob_file, filename) else: try: info_entity = self.store_blob(content_type, filename, digester, blob_file, creation) except _TooManyConflictsError: too_many_conflicts = True break # Track created blobs in case we need to roll them back. created_blobs.append(info_entity) variable.add_header('Content-Type', 'message/external-body', access_type=blobstore.BLOB_KEY_HEADER, blob_key=info_entity.key().name()) variable.set_payload([external]) # Set common information. variable.add_header('Content-Disposition', 'form-data', **disposition_parameters) message.attach(variable) if (mime_type_error or too_many_conflicts or upload_too_large or filename_too_large or content_type_too_large): for blob in created_blobs: datastore.Delete(blob) if mime_type_error: self.abort(400, detail=mime_type_error) elif too_many_conflicts: self.abort(500, detail='Could not generate a blob key.') elif upload_too_large: self.abort(413) else: if filename_too_large: invalid_field = 'filename' elif content_type_too_large: invalid_field = 'Content-Type' detail = 'The %s exceeds the maximum allowed length of %s.' % ( invalid_field, _MAX_STRING_NAME_LENGTH) self.abort(400, detail=detail) message_out = cStringIO.StringIO() gen = email.generator.Generator(message_out, maxheaderlen=0) gen.flatten(message, unixfrom=False) # Get the content text out of the message. message_text = message_out.getvalue() content_start = message_text.find('\n\n') + 2 content_text = message_text[content_start:] content_text = content_text.replace('\n', '\r\n') return message.get('Content-Type'), content_text def store_blob_and_transform_request(self, environ): """Stores a blob in response to a WSGI request and transforms environ. environ is modified so that it is suitable for forwarding to the user's application. Args: environ: An environ dict for the current request as defined in PEP-333. Raises: webob.exc.HTTPException: The upload failed. """ # Only permit POST. if environ['REQUEST_METHOD'].lower() != 'post': self.abort(405) url_match = _UPLOAD_URL_PATTERN.match(environ['PATH_INFO']) if not url_match: self.abort(404) upload_key = url_match.group(1) # Retrieve upload session. try: upload_session = datastore.Get(upload_key) except datastore_errors.EntityNotFoundError: detail = 'No such upload session: %s' % upload_key logging.error(detail) self.abort(404, detail=detail) success_path = upload_session['success_path'].encode('ascii') max_bytes_per_blob = upload_session['max_bytes_per_blob'] max_bytes_total = upload_session['max_bytes_total'] bucket_name = upload_session.get('gs_bucket_name', None) upload_form = cgi.FieldStorage(fp=environ['wsgi.input'], environ=environ) # Generate debug log message logstrings = [] for k in sorted(upload_form): vs = upload_form[k] if not isinstance(vs, list): vs = [vs] for v in vs: if v.filename: logstrings.append('%s=%s' % (k, v.filename)) logging.debug('Received blobstore upload: %s', ', '.join(logstrings)) # It's ok to read the whole string in memory because the content is # merely a reference to the blob, not the blob itself. content_type, content_text = self.store_and_build_forward_message( upload_form, max_bytes_per_blob=max_bytes_per_blob, max_bytes_total=max_bytes_total, bucket_name=bucket_name) datastore.Delete(upload_session) # Ensure that certain HTTP_ variables are not forwarded. for name in _STRIPPED_ENVIRON: if name in environ: del environ[name] # Replace some HTTP headers in the forwarded environ. environ['CONTENT_TYPE'] = content_type environ['CONTENT_LENGTH'] = str(len(content_text)) # Forward on to success_path. Like production, only the path and query # matter. parsed_url = urlparse.urlsplit(success_path) environ['PATH_INFO'] = parsed_url.path if parsed_url.query: environ['QUERY_STRING'] = parsed_url.query # The user is always an administrator for the forwarded request. environ[constants.FAKE_IS_ADMIN_HEADER] = '1' # Set the wsgi variables environ['wsgi.input'] = cStringIO.StringIO(content_text) def __call__(self, environ, start_response): """Handles WSGI requests. Args: environ: An environ dict for the current request as defined in PEP-333. start_response: A function with semantics defined in PEP-333. Returns: An iterable over strings containing the body of the HTTP response. """ # Handle any errors in the blob uploader, but do not catch errors raised by # the user's application. try: self.store_blob_and_transform_request(environ) except webob.exc.HTTPException, e: def start_response_with_exc_info(status, headers, exc_info=sys.exc_info()): start_response(status, headers, exc_info) return e(environ, start_response_with_exc_info) return self._forward_app(environ, start_response)