#!/usr/bin/env python # # Copyright 2007 Google Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # """External script for generating Cloud Endpoints related files. The gen_discovery_doc subcommand takes a list of fully qualified ProtoRPC service names and calls a cloud service which generates a discovery document in REST or RPC style. Example: endpointscfg.py gen_discovery_doc -o . -f rest postservice.GreetingsV1 The gen_client_lib subcommand takes a discovery document and calls a cloud service to generate a client library for a target language (currently just Java) Example: endpointscfg.py gen_client_lib java -o . greetings-v0.1.discovery The get_client_lib subcommand does both of the above commands at once. Example: endpointscfg.py get_client_lib java -o . postservice.GreetingsV1 The gen_api_config command outputs an .api configuration file for a service. Example: endpointscfg.py gen_api_config -o . -a /path/to/app \ --hostname myhost.appspot.com postservice.GreetingsV1 """ from __future__ import with_statement import argparse import collections import contextlib try: import json except ImportError: import simplejson as json import os import re import sys import urllib import urllib2 from endpoints import api_config from protorpc import remote import yaml from google.appengine.tools.devappserver2 import api_server DISCOVERY_DOC_BASE = ('https://webapis-discovery.appspot.com/_ah/api/' 'discovery/v1/apis/generate/') CLIENT_LIBRARY_BASE = 'https://google-api-client-libraries.appspot.com/generate' _VISIBLE_COMMANDS = ('get_client_lib', 'get_discovery_doc') class ServerRequestException(Exception): """Exception for problems with the request to a server.""" def __init__(self, http_error): """Create a ServerRequestException from a given urllib2.HTTPError. Args: http_error: The HTTPError that the ServerRequestException will be based on. """ error_details = None error_response = None if http_error.fp: try: error_response = http_error.fp.read() error_body = json.loads(error_response) error_details = ['%s: %s' % (detail['message'], detail['debug_info']) for detail in error_body['error']['errors']] except (ValueError, TypeError, KeyError): pass if error_details: error_details_str = ', '.join(error_details) error_message = ('HTTP %s (%s) error when communicating with URL: %s. ' 'Details: %s' % (http_error.code, http_error.reason, http_error.filename, error_details_str)) else: error_message = ('HTTP %s (%s) error when communicating with URL: %s. ' 'Response: %s' % (http_error.code, http_error.reason, http_error.filename, error_response)) super(ServerRequestException, self).__init__(error_message) class _EndpointsParser(argparse.ArgumentParser): """Create a subclass of argparse.ArgumentParser for Endpoints.""" def error(self, message): """Override superclass to support customized error message. Error message needs to be rewritten in order to display visible commands only, when invalid command is called by user. Otherwise, hidden commands will be displayed in stderr, which is not expected. Refer the following argparse python documentation for detailed method information: http://docs.python.org/2/library/argparse.html#exiting-methods Args: message: original error message that will be printed to stderr """ subcommands_quoted = ', '.join( [repr(command) for command in _VISIBLE_COMMANDS]) subcommands = ', '.join(_VISIBLE_COMMANDS) message = re.sub( r'(argument {%s}: invalid choice: .*) \(choose from (.*)\)$' % subcommands, r'\1 (choose from %s)' % subcommands_quoted, message) super(_EndpointsParser, self).error(message) def _WriteFile(output_path, name, content): """Write given content to a file in a given directory. Args: output_path: The directory to store the file in. name: The name of the file to store the content in. content: The content to write to the file.close Returns: The full path to the written file. """ path = os.path.join(output_path, name) with open(path, 'wb') as f: f.write(content) return path def GenApiConfig(service_class_names, config_string_generator=None, hostname=None, application_path=None): """Write an API configuration for endpoints annotated ProtoRPC services. Args: service_class_names: A list of fully qualified ProtoRPC service classes. config_string_generator: A generator object that produces API config strings using its pretty_print_config_to_json method. hostname: A string hostname which will be used as the default version hostname. If no hostname is specificied in the @endpoints.api decorator, this value is the fallback. application_path: A string with the path to the AppEngine application. Raises: TypeError: If any service classes don't inherit from remote.Service. messages.DefinitionNotFoundError: If a service can't be found. Returns: A map from service names to a string containing the API configuration of the service in JSON format. """ api_service_map = collections.OrderedDict() for service_class_name in service_class_names: module_name, base_service_class_name = service_class_name.rsplit('.', 1) module = __import__(module_name, fromlist=base_service_class_name) service = getattr(module, base_service_class_name) if not isinstance(service, type) or not issubclass(service, remote.Service): raise TypeError('%s is not a ProtoRPC service' % service_class_name) services = api_service_map.setdefault( (service.api_info.name, service.api_info.version), []) services.append(service) app_yaml_hostname = _GetAppYamlHostname(application_path) service_map = collections.OrderedDict() config_string_generator = ( config_string_generator or api_config.ApiConfigGenerator()) for api_info, services in api_service_map.iteritems(): assert len(services) > 0, 'An API must have at least one ProtoRPC service' hostname = services[0].api_info.hostname or hostname or app_yaml_hostname service_map['%s-%s' % api_info] = ( config_string_generator.pretty_print_config_to_json( services, hostname=hostname)) return service_map def _GetAppYamlHostname(application_path, open_func=open): """Build the hostname for this app based on the name in app.yaml. Args: application_path: A string with the path to the AppEngine application. This should be the directory containing the app.yaml file. open_func: Function to call to open a file. Used to override the default open function in unit tests. Returns: A hostname, usually in the form of "myapp.appspot.com", based on the application name in the app.yaml file. If the file can't be found or there's a problem building the name, this will return None. """ try: app_yaml_file = open_func(os.path.join(application_path or '.', 'app.yaml')) config = yaml.safe_load(app_yaml_file.read()) except IOError: return None application = config.get('application') if not application: return None if ':' in application: return None tilde_index = application.rfind('~') if tilde_index >= 0: application = application[tilde_index + 1:] if not application: return None return '%s.appspot.com' % application def _FetchDiscoveryDoc(config, doc_format): """Fetch discovery documents generated from a cloud service. Args: config: An API config. doc_format: The requested format for the discovery doc. (rest|rpc) Raises: ServerRequestException: If fetching the generated discovery doc fails. Returns: A list of discovery doc strings. """ body = json.dumps({'config': config}, indent=2, sort_keys=True) request = urllib2.Request(DISCOVERY_DOC_BASE + doc_format, body) request.add_header('content-type', 'application/json') try: with contextlib.closing(urllib2.urlopen(request)) as response: return response.read() except urllib2.HTTPError, error: raise ServerRequestException(error) def _GenDiscoveryDoc(service_class_names, doc_format, output_path, hostname=None, application_path=None): """Write discovery documents generated from a cloud service to file. Args: service_class_names: A list of fully qualified ProtoRPC service names. doc_format: The requested format for the discovery doc. (rest|rpc) output_path: The directory to output the discovery docs to. hostname: A string hostname which will be used as the default version hostname. If no hostname is specificied in the @endpoints.api decorator, this value is the fallback. Defaults to None. application_path: A string containing the path to the AppEngine app. Raises: ServerRequestException: If fetching the generated discovery doc fails. Returns: A list of discovery doc filenames. """ output_files = [] service_configs = GenApiConfig(service_class_names, hostname=hostname, application_path=application_path) for api_name_version, config in service_configs.iteritems(): discovery_doc = _FetchDiscoveryDoc(config, doc_format) discovery_name = api_name_version + '.discovery' output_files.append(_WriteFile(output_path, discovery_name, discovery_doc)) return output_files def _GenClientLib(discovery_path, language, output_path, build_system): """Write a client library from a discovery doc, using a cloud service to file. Args: discovery_path: Path to the discovery doc used to generate the client library. language: The client library language to generate. (java) output_path: The directory to output the client library zip to. build_system: The target build system for the client library language. Raises: IOError: If reading the discovery doc fails. ServerRequestException: If fetching the generated client library fails. Returns: The path to the zipped client library. """ with open(discovery_path) as f: discovery_doc = f.read() client_name = re.sub(r'\.discovery$', '.zip', os.path.basename(discovery_path)) return _GenClientLibFromContents(discovery_doc, language, output_path, build_system, client_name) def _GenClientLibFromContents(discovery_doc, language, output_path, build_system, client_name): """Write a client library from a discovery doc, using a cloud service to file. Args: discovery_doc: A string, the contents of the discovery doc used to generate the client library. language: A string, the client library language to generate. (java) output_path: A string, the directory to output the client library zip to. build_system: A string, the target build system for the client language. client_name: A string, the filename used to save the client lib. Raises: IOError: If reading the discovery doc fails. ServerRequestException: If fetching the generated client library fails. Returns: The path to the zipped client library. """ body = urllib.urlencode({'lang': language, 'content': discovery_doc, 'layout': build_system}) request = urllib2.Request(CLIENT_LIBRARY_BASE, body) try: with contextlib.closing(urllib2.urlopen(request)) as response: content = response.read() return _WriteFile(output_path, client_name, content) except urllib2.HTTPError, error: raise ServerRequestException(error) def _GetClientLib(service_class_names, language, output_path, build_system, hostname=None, application_path=None): """Fetch client libraries from a cloud service. Args: service_class_names: A list of fully qualified ProtoRPC service names. language: The client library language to generate. (java) output_path: The directory to output the discovery docs to. build_system: The target build system for the client library language. hostname: A string hostname which will be used as the default version hostname. If no hostname is specificied in the @endpoints.api decorator, this value is the fallback. Defaults to None. application_path: A string containing the path to the AppEngine app. Returns: A list of paths to client libraries. """ client_libs = [] service_configs = GenApiConfig(service_class_names, hostname=hostname, application_path=application_path) for api_name_version, config in service_configs.iteritems(): discovery_doc = _FetchDiscoveryDoc(config, 'rest') client_name = api_name_version + '.zip' client_libs.append( _GenClientLibFromContents(discovery_doc, language, output_path, build_system, client_name)) return client_libs def _GenApiConfigCallback(args, api_func=GenApiConfig): """Generate an api file. Args: args: An argparse.Namespace object to extract parameters from. api_func: A function that generates and returns an API configuration for a list of services. """ service_configs = api_func(args.service, hostname=args.hostname, application_path=args.application) for api_name_version, config in service_configs.iteritems(): _WriteFile(args.output, api_name_version + '.api', config) def _GetClientLibCallback(args, client_func=_GetClientLib): """Generate discovery docs and client libraries to files. Args: args: An argparse.Namespace object to extract parameters from. client_func: A function that generates client libraries and stores them to files, accepting a list of service names, a client library language, an output directory, a build system for the client library language, and a hostname. """ client_paths = client_func( args.service, args.language, args.output, args.build_system, hostname=args.hostname, application_path=args.application) for client_path in client_paths: print 'API client library written to %s' % client_path def _GenDiscoveryDocCallback(args, discovery_func=_GenDiscoveryDoc): """Generate discovery docs to files. Args: args: An argparse.Namespace object to extract parameters from discovery_func: A function that generates discovery docs and stores them to files, accepting a list of service names, a discovery doc format, and an output directory. """ discovery_paths = discovery_func(args.service, args.format, args.output, hostname=args.hostname, application_path=args.application) for discovery_path in discovery_paths: print 'API discovery document written to %s' % discovery_path def _GenClientLibCallback(args, client_func=_GenClientLib): """Generate a client library to file. Args: args: An argparse.Namespace object to extract parameters from client_func: A function that generates client libraries and stores them to files, accepting a path to a discovery doc, a client library language, an output directory, and a build system for the client library language. """ client_path = client_func(args.discovery_doc[0], args.language, args.output, args.build_system) print 'API client library written to %s' % client_path def MakeParser(prog): """Create an argument parser. Args: prog: The name of the program to use when outputting help text. Returns: An argparse.ArgumentParser built to specification. """ def AddStandardOptions(parser, *args): """Add common endpoints options to a parser. Args: parser: The parser to add options to. *args: A list of option names to add. Possible names are: application, format, output, language, service, and discovery_doc. """ if 'application' in args: parser.add_argument('-a', '--application', default='.', help='The path to the Python App Engine App') if 'format' in args: parser.add_argument('-f', '--format', default='rest', choices=['rest', 'rpc'], help='The requested API protocol type') if 'hostname' in args: help_text = ('Default application hostname, if none is specified ' 'for API service.') parser.add_argument('--hostname', help=help_text) if 'output' in args: parser.add_argument('-o', '--output', default='.', help='The directory to store output files') if 'language' in args: parser.add_argument('language', help='The target output programming language') if 'service' in args: parser.add_argument('service', nargs='+', help='Fully qualified service class name') if 'discovery_doc' in args: parser.add_argument('discovery_doc', nargs=1, help='Path to the discovery document') if 'build_system' in args: parser.add_argument('-bs', '--build_system', default='default', help='The target build system') parser = _EndpointsParser(prog=prog) subparsers = parser.add_subparsers( title='subcommands', metavar='{%s}' % ', '.join(_VISIBLE_COMMANDS)) get_client_lib = subparsers.add_parser( 'get_client_lib', help=('Generates discovery documents and client ' 'libraries from service classes')) get_client_lib.set_defaults(callback=_GetClientLibCallback) AddStandardOptions(get_client_lib, 'application', 'hostname', 'output', 'language', 'service', 'build_system') get_discovery_doc = subparsers.add_parser( 'get_discovery_doc', help='Generates discovery documents from service classes') get_discovery_doc.set_defaults(callback=_GenDiscoveryDocCallback) AddStandardOptions(get_discovery_doc, 'application', 'format', 'hostname', 'output', 'service') gen_api_config = subparsers.add_parser('gen_api_config') gen_api_config.set_defaults(callback=_GenApiConfigCallback) AddStandardOptions(gen_api_config, 'application', 'hostname', 'output', 'service') gen_discovery_doc = subparsers.add_parser('gen_discovery_doc') gen_discovery_doc.set_defaults(callback=_GenDiscoveryDocCallback) AddStandardOptions(gen_discovery_doc, 'application', 'format', 'hostname', 'output', 'service') gen_client_lib = subparsers.add_parser('gen_client_lib') gen_client_lib.set_defaults(callback=_GenClientLibCallback) AddStandardOptions(gen_client_lib, 'output', 'language', 'discovery_doc', 'build_system') return parser def main(argv): api_server.test_setup_stubs(app_id='_') parser = MakeParser(argv[0]) args = parser.parse_args(argv[1:]) application_path = getattr(args, 'application', None) if application_path is not None: sys.path.insert(0, os.path.abspath(application_path)) args.callback(args) if __name__ == '__main__': main(sys.argv)