# Atlas Configuration Apache Atlas Metadata Server Configuration. Internally Atlas embeds a [Titan][titan] graph database. Titan for working, needs a solr cluster for storing indexes and an HBase cluster as the data storage. Atlas needs also kafka as a bus to broadcats message betwwen the different components (e.g. Hive, Ranger). module.exports = (service) -> options = service.options ## Identities options.hadoop_group = merge {}, service.deps.hadoop_core.options.hadoop_group, options.hadoop_group # Group options.group = name: options.group if typeof options.group is 'string' options.group ?= {} options.group.name ?= 'atlas' options.group.system ?= true # User options.user = name: options.user if typeof options.user is 'string' options.user ?= {} options.user.name ?= 'atlas' options.user.system ?= true options.user.comment ?= 'Atlas User' options.user.home ?= '/var/lib/atlas' options.user.groups ?= ['hadoop'] options.user.gid = options.group.name options.hbase_admin = service.deps.hbase_master[0].options.admin options.kafka_admin = service.deps.kafka_broker[0].options.admin ## Access options.ranger_kafka_install = service.deps.ranger_kafka[0].options.install if service.deps.ranger_kafka options.ranger_hbase_install = service.deps.ranger_hbase[0].options.install if service.deps.ranger_hbase # options.ranger_solr_install = service.deps.ranger_solr[0].options.install if service.deps.ranger_solr options.ranger_admin = service.deps.ranger_admin options.hbase_conf_dir = service.deps.hbase_client.options.conf_dir options.solr_cloud = service.deps.solr_cloud[0] if service.deps.solr_cloud ## Kerberos options.krb5 ?= {} options.krb5.realm ?= service.deps.krb5_client.options.etc_krb5_conf?.libdefaults?.default_realm throw Error 'Required Options: "realm"' unless options.krb5.realm options.krb5.admin ?= service.deps.krb5_client.options.admin[options.krb5.realm] ## Environment options.conf_dir ?= '/etc/atlas/conf' options.log_dir ?= '/var/log/atlas' options.pid_dir ?= '/var/run/atlas' options.server_opts ?= '' options.server_heap ?= '' options.cluster_name ?= 'atlas-on-ryba-cluster' options.metadata_opts ?= {} # options.metadata_opts['java.security.auth.login.config'] ?= "#{options.conf_dir}/atlas-server.jaas" options.atlas_opts ?= {} options.atlas_opts['java.security.auth.login.config'] ?= "#{options.conf_dir}/atlas-server.jaas" options.atlas_opts['log4j.configuration'] ?= 'atlas-log4j.xml' # Layout options.env ?= {} options.env['ATLAS_PID_DIR'] ?= "#{options.pid_dir}" options.env['ATLAS_LOG_DIR'] ?= "#{options.log_dir}" options.env['ATLAS_HOME_DIR'] ?= '/usr/hdp/current/atlas-server' options.env['ATLAS_DATA_DIR'] ?= "#{options.user.home}/data" options.env['ATLAS_EXPANDED_WEBAPP_DIR'] ?= "#{options.user.home}/server/webapp" options.env['HBASE_CONF_DIR'] ?= "#{options.conf_dir}/hbase" # Java options.min_heap ?= '512m' options.max_heap ?= '512m' options.iptables ?= service.deps.iptables and service.deps.iptables.options.action is 'start' # Misc options.clean_logs ?= false ## Configuration options.fqdn ?= service.node.fqdn options.shortname ?= service.node.hostname options.application ?= {} options.application.properties ?= {} options.application.properties['atlas.server.bind.address'] ?= "#{service.node.fqdn}" # migration: wdavidw, really, bindinng a server to a FQDN and not an IP or 0.0.0.0 ? let's leave a comment to explain it. options.application.properties['atlas.taxonomy.default.name'] ?= 'Catalog' options.application.properties['atlas.rest-csrf.enabled'] ?= 'true' options.application.properties['atlas.rest-csrf.browser-useragents-regex'] ?= '^Mozilla.*,^Opera.*,^Chrome.*' options.application.properties['atlas.rest-csrf.methods-to-ignore'] ?= 'GET,OPTIONS,HEAD,TRACE' options.application.properties['atlas.rest-csrf.custom-header'] ?= 'X-XSRF-HEADER' options.application.properties['atlas.feature.taxonomy.enable'] ?= 'true' ## Kerberos Atlas, when communicating with other components (Solr,HBase,Kafka) needs to authenticate itself as a client. It uses the JAAS mechanism. The JAAS informations can be set via a jaas file or the properties can be set directly from atlas-application.properties file. options.application.properties['atlas.authentication.method'] ?= service.deps.hadoop_core.options.core_site['hadoop.security.authentication'] if options.application.properties['atlas.authentication.method'] is 'kerberos' options.application.properties['atlas.authentication.principal'] ?= "#{options.user.name}/_HOST@#{options.krb5.realm}" options.application.properties['atlas.authentication.keytab'] ?= '/etc/security/keytabs/atlas.service.keytab' options.application.properties['atlas.http.authentication.enabled'] ?= true options.application.properties['atlas.http.authentication.type'] ?= options.application.properties['atlas.authentication.method'] options.application.properties['atlas.http.authentication.kerberos.principal'] ?= "HTTP/_HOST@#{options.krb5.realm}" options.application.properties['atlas.http.authentication.kerberos.keytab'] ?= '/etc/security/keytabs/spnego.service.keytab' # options.application.properties['atlas.jaas.Client.loginModuleName'] ?= 'com.sun.security.auth.module.Krb5LoginModule' # options.application.properties['atlas.jaas.Client.loginModuleControlFlag'] ?= 'required' # options.application.properties['atlas.jaas.Client.option.useKeyTab'] ?= 'true' # options.application.properties['atlas.jaas.Client.option.storeKey'] ?= 'true' # options.application.properties['atlas.jaas.Client.option.useTicketCache'] ?= 'false' # options.application.properties['atlas.jaas.Client.option.doNotPrompt'] ?= 'false' # options.application.properties['atlas.jaas.Client.option.keyTab'] ?= options.application.properties['atlas.authentication.keytab'] # options.application.properties['atlas.jaas.Client.option.principal'] ?= options.application.properties['atlas.authentication.principal'].replace '_HOST', service.node.fqdn match_princ = /^(.+?)[@\/]/.exec options.application.properties['atlas.authentication.principal'] throw Error 'Invalid Atlas principal' unless match_princ? options.admin_principal ?= "#{options.user.name}@#{options.krb5.realm}" throw Error 'Missing Atlas Admin Principal Password admin_password' unless options.admin_password? match_admin = /^(.+?)[@\/]/.exec options.admin_principal throw Error "Principal Name does not match admin user name" unless match_admin[1] is match_princ[1] ## Authentication Method Several authentication method can be used (Kerberos, ldap, file). More that one method can be enabled. An admin user should exist for managinf other. Note: In HDP 2.5.x LDAP option in Atlas can't pass full DN parameters which resultat in impossibility to log in using LDAP: Logs can look like `[LDAP: error code 32 - No Such Object]` The solution is to escape all commas in the DNs used for LDAP configuration. Forexample, `atlas.authentication.method.ldap.userDNpattern`=`cn=users\,cn=accounts\,dc=field\,dc=hortonworks\,dc=com`. options.application.properties['atlas.authentication.method.ldap'] ?= 'false' #No custom configs options.application.properties['atlas.authentication.method.kerberos'] ?= "#{service.deps.hadoop_core.options.core_site['hadoop.security.authentication'] is 'kerberos'}" options.application.properties['atlas.authentication.method.file'] ?= 'true' # Configure kerberos authentication if options.application.properties['atlas.authentication.method.kerberos'] is 'true' options.application.properties['atlas.authentication.method.kerberos.principal'] ?= options.application.properties['atlas.http.authentication.kerberos.principal'] options.application.properties['atlas.authentication.method.kerberos.keytab'] ?= options.application.properties['atlas.http.authentication.kerberos.keytab'] # options.application.properties['atlas.authentication.method.kerberos.name.rules'] ?= core_site['hadoop.security.auth_to_local'] # Configure file authentication if options.application.properties['atlas.authentication.method.file'] is 'true' options.application.properties['atlas.authentication.method.file.filename'] ?= "#{options.conf_dir}/users-credentials.properties" options.admin_user ?= 'admin' options.admin_password ?= 'admin123' options.user_creds ?= {} options.user_creds["#{options.admin_user}"] ?= name: "#{options.admin_user}" password: "#{options.admin_password}" group: "ADMIN" ## Authorization Atlas accepts only simple (file) or Ranger based [authorization](http://atlas.incubator.apache.org/Authentication-Authorization.html). # for now ranger tagsync does not exist # options.application.properties['atlas.authorizer.impl'] ?= if service.deps.ranger_tagsync.length > 0 then 'ranger' else 'simple' options.application.properties['atlas.authorizer.impl'] ?= 'simple' if options.application.properties['atlas.authorizer.impl'] is 'simple' options.application.properties['atlas.auth.policy.file'] ?= "#{options.conf_dir}/policy-store.txt" options.admin_users ?= [] options.admin_users.push options.admin_users if options.admin_user? and options.admin_users.indexOf(options.admin_user) isnt -1 ## Automatic Check on start up Atlas server does take care of parallel executions of the setup steps. options.application.properties['atlas.server.run.setup.on.start'] ?= 'false' ## SSL Atlas SSL Encryption can be enabled by configuring following properties. options.ssl = merge {}, service.deps.ssl?.options, truststore: target: "#{options.conf_dir}/truststore" keystore: target: "#{options.conf_dir}/keystore" , options.ssl options.ssl.enabled ?= !!service.deps.ssl if options.ssl.enabled throw Error "Required Option: ssl.cert" if not options.ssl.cert throw Error "Required Option: ssl.key" if not options.ssl.key throw Error "Required Option: ssl.cacert" if not options.ssl.cacert options.application.properties['atlas.enableTLS'] ?= 'true' # Truststore options.application.properties['truststore.file'] ?= options.ssl.truststore.target throw Error "Required Property: truststore.password" if not options.ssl.truststore.password # Keystore throw Error 'Missing ssl.keystore.keypass' unless options.ssl.keystore.keypass options.application.properties['keystore.file'] ?= options.ssl.keystore.target throw Error "Required Property: keystore.password" if not options.ssl.keystore.password options.application.properties['client.auth.enabled'] ?= "false" options.application.properties['atlas.server.http.port'] ?= '21000' options.application.properties['atlas.server.https.port'] ?= '21443' # http://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/CommandsManual.html#credential # for now ryba only support creating the jceks file on the local atlas server. # but you can manually (if using the same truststore and keystore on each server) # uploading the provider generate into hdfs options.application.properties['cert.stores.credential.provider.path'] ?= "jceks://file#{options.conf_dir}/credential.jceks" protocol = if options.application.properties['atlas.enableTLS'] is 'true' then 'https' else 'http' port = options.application.properties["atlas.server.#{protocol}.port"] options.application.properties['atlas.app.port'] ?= port ## REST Address rest_address = "#{protocol}://#{service.node.fqdn}:#{port}" options.application.properties['atlas.rest.address'] ?= rest_address options.application.urls ?= "#{rest_address}" ## High Availability From 0.7.0 (hdp 2.5.3.0-37) Atlas Metada server support high availability in an active/passive fashion. Failover is automatically based on Zookeeper leader election. The Quorum property is mandatory even if the HA is not enabled or the following error is thrown Error injecting constructor, java.lang.NullPointerException: connectionString cannot be null zookeeper_quorum = for srv in service.deps.zookeeper_server continue unless srv.options.config['peerType'] is 'participant' "#{srv.node.fqdn}:#{srv.options.config['clientPort']}" options.application.properties['atlas.server.ha.zookeeper.connect'] ?= zookeeper_quorum.join ',' if service.deps.atlas.length > 1 options.application.properties['atlas.server.ha.zookeeper.acl'] ?= "sasl:#{options.user.name}@#{options.krb5.realm}" options.application.properties['atlas.server.ha.zookeeper.auth'] ?= "sasl:#{options.user.name}@#{options.krb5.realm}" options.application.properties['atlas.server.ha.zookeeper.zkroot'] ?= '/apache_atlas' ## High Availability with Automatic Failover Using Official atlas [documentation](http://atlas.incubator.apache.org/HighAvailability.html) for HA configuration, (not available on HDP website). Ryba does configure atlas server with the same port for every instance. options.application.properties['atlas.server.ha.enabled'] ?= if service.deps.atlas.length > 1 then 'true' else 'false' options.application.properties['atlas.server.ids'] ?= service.instances.map( (instance) -> instance.node.hostname ).join ',' for node in service.instances.map( (instance) -> instance.node) options.application.properties["atlas.server.address.#{node.hostname}"] ?= "#{node.fqdn}:#{port}" ## Atlas Ranger User options.ranger_user ?= "name": 'atlas' "firstName": 'atlas' "lastName": 'hadoop' "emailAddress": 'atlas@hadoop.ryba' 'userSource': 1 'userRoleList': ['ROLE_USER'] 'groups': [] 'status': 1 ## Apache Kafka Notification Required for Ranger integration or anytime there is a consumer of entity change notifications. if service.deps.kafka_broker? # Includes the required Atlas directories to the hadoop-env configuration export service.deps.hadoop_core.options.hadoop_classpath = add_prop service.deps.hadoop_core.options.hadoop_classpath, "#{options.conf_dir}:/usr/hdp/current/atlas-server/hook/hive", ':' # Configure Kafka Broker properties options.application.properties['atlas.notification.embedded'] ?= 'false' options.application.properties['atlas.kafka.sasl.kerberos.service.name'] ?= 'kafka' options.application.properties['atlas.kafka.data'] ?= "#{options.user.home}/data/kafka" # {options.user.home}/keystore" options.application.properties['atlas.kafka.zookeeper.connect'] ?= "#{service.deps.kafka_broker[0].options.config['zookeeper.connect']}" options.application.properties['atlas.kafka.zookeeper.session.timeout.ms'] ?= '1000' options.application.properties['atlas.kafka.zookeeper.sync.time.ms'] ?= '20' options.application.properties['atlas.kafka.auto.commit.interval.ms'] ?= '1000' options.application.properties['atlas.kafka.hook.group.id'] ?= 'atlas' options.application.properties['atlas.kafka.entities.group.id'] ?= 'atlas' options.application.properties['atlas.kafka.auto.commit.enable'] ?= 'false' options.application.properties['atlas.notification.create.topics'] ?= 'false' options.application.properties['atlas.notification.replicas'] ?= "#{service.deps.kafka_broker.length}" options.application.properties['atlas.notification.log.failed.messages'] ?= '500' options.application.properties['atlas.notification.consumer.retry.interval'] ?= '10' options.application.properties['atlas.notification.hook.retry.interval'] ?= '1000' options.application.properties['atlas.kafka.auto.offset.reset'] ?= 'smallest' options.application.properties['atlas.notification.topics'] ?= 'ATLAS_HOOK,ATLAS_ENTITIES' # Kafka Jaas client configuration options.application.properties['atlas.jaas.KafkaClient.loginModuleControlFlag'] ?= 'required' options.application.properties['atlas.jaas.KafkaClient.loginModuleName'] ?= 'com.sun.security.auth.module.Krb5LoginModule' options.application.properties['atlas.jaas.KafkaClient.option.keyTab'] ?= '/etc/security/keytabs/atlas.service.keytab' options.application.properties['atlas.jaas.KafkaClient.option.principal'] ?= "#{options.user.name}/_HOST@#{options.krb5.realm}" options.application.properties['atlas.jaas.KafkaClient.option.serviceName'] ?= 'kafka' options.application.properties['atlas.jaas.KafkaClient.option.storeKey'] ?= 'true' options.application.properties['atlas.jaas.KafkaClient.option.useKeyTab'] ?= 'true' options.kafka_partitions ?= Math.max(service.deps.kafka_broker.length-1,1) options.kafka_replication ?= Math.max(service.deps.kafka_broker.length-1,1) # Choose kafka broker channel by preference order chanels = [] chanels.push 'SASL_SSL' if service.deps.hadoop_core.options.core_site['hadoop.security.authentication'] is 'kerberos' and options.application.properties['atlas.enableTLS'] is 'true' chanels.push 'SASL_PLAINTEXT' if service.deps.hadoop_core.options.core_site['hadoop.security.authentication'] is 'kerberos' chanels.push 'SSL' if options.application.properties['atlas.enableTLS'] is 'true' chanels.push 'PLAINTEXT' # Recording choosen protocol options.application.properties['atlas.kafka.security.protocol'] ?= chanels[0] options.application.kafka_chanel = options.application.properties['atlas.kafka.security.protocol'] # `kafka.broker.protocols` are available client protocols for communicating with broker if options.application.kafka_chanel in service.deps.kafka_broker[0].options.protocols brokers = service.deps.kafka_broker.map( (srv) => "#{srv.node.fqdn}:#{srv.options.ports[options.application.kafka_chanel]}" ).join ',' # construcut the bootstrap listeners string base on channel # i.e.: SASL_SSL://master1.ryba:9096,master2.ryba:9096,master3.ryba:9096 for example options.application.properties['atlas.kafka.bootstrap.servers'] ?= "#{options.application.kafka_chanel}://#{brokers}" if options.application.kafka_chanel in ['SSL','SASL_SSL'] options.application.properties['atlas.kafka.ssl.truststore.location'] ?= service.deps.kafka_client.options.consumer.config['ssl.truststore.location'] options.application.properties['atlas.kafka.ssl.truststore.password'] ?= service.deps.kafka_client.options.consumer.config['ssl.truststore.password'] else throw Error "Atlas Selected Kafka Protocol #{options.application.kafka_chanel} is not allowed by Kafka Brokers configuration" #Kafka Ranger Plugin if options.ranger_kafka_install users = ["#{options.user.name}"] users.push 'ANONYMOUS' if (options.application.properties['atlas.kafka.security.protocol'] in ['PLAINTEXT','SSL']) options.kafka_policy = service: "#{options.ranger_kafka_install['REPOSITORY_NAME']}" name: "atlas-metadata-server-#{service.node.fqdn}" description: "Atlas MetaData Server ACL" isAuditEnabled: true resources: topic: values: options.application.properties['atlas.notification.topics'] isExcludes: false isRecursive: false 'policyItems': [ "accesses": [ 'type': 'publish' 'isAllowed': true , 'type': 'consume' 'isAllowed': true , 'type': 'configure' 'isAllowed': true , 'type': 'describe' 'isAllowed': true , 'type': 'create' 'isAllowed': true , 'type': 'delete' 'isAllowed': true , 'type': 'kafka_admin' 'isAllowed': true ], 'users': users 'groups': [] 'conditions': [] 'delegateAdmin': true ] # ## Ranger Tag Base Policies configuration # # for srv in service.deps.ranger_tagsync # srv.options.atlas_properties ?= {} # for prop in [ # 'atlas.notification.embedded' # 'atlas.kafka.data' # 'atlas.kafka.zookeeper.connect' # 'atlas.kafka.bootstrap.servers' # 'atlas.kafka.security.protocol' # 'atlas.kafka.zookeeper.session.timeout.ms' # 'atlas.kafka.zookeeper.sync.time.ms' # 'atlas.kafka.auto.commit.interval.ms' # 'atlas.kafka.hook.group.id' # ] then srv.options.atlas_properties[prop] ?= options.application.properties[prop] ## Titan MetaData Database Atlas uses Titan as its its metadata storage. The Titan database can be configured with HBase as its storage database, and Solr for its indexing backend ### Storage Backend Ryba does only support HBase storage backend for now. It configures policies in case ranger is enabled on the cluster options.storage_engine ?= 'hbase' if options.storage_engine is 'hbase' #HBase configurations if service.deps.hbase_master options.application.properties['atlas.graph.storage.backend'] ?= 'hbase' options.application.properties['atlas.graph.storage.hostname'] ?= service.deps.hbase_master[0].options.hbase_site['hbase.zookeeper.quorum'] options.application.properties['zookeeper.znode.parent'] ?= service.deps.hbase_master[0].options.hbase_site['zookeeper.znode.parent'] options.application.namespace ?= 'atlas' options.application.properties['atlas.graph.storage.hbase.table'] ?= "#{options.application.namespace}:atlas_titan" options.application.properties['atlas.audit.hbase.tablename'] ?= "#{options.application.namespace}:atlas_entity_audit" else throw Error 'No HBase cluster is configured' #Ranger HBase Plugin configuration options.hbase_policy = "name": "Atlas-Titan-to-HBase-policy" "service": "#{options.ranger_hbase_install['REPOSITORY_NAME']}" "resources": "column": "values": ["*"] "isExcludes": false "isRecursive": false "column-family": "values": ["*"] "isExcludes": false "isRecursive": false "table": "values": [ "#{options.application.properties['atlas.graph.storage.hbase.table']}", "#{options.application.properties['atlas.audit.hbase.tablename']}" ] "isExcludes": false "isRecursive": false "repositoryName": "#{options.ranger_hbase_install['REPOSITORY_NAME']}" "repositoryType": "hbase" "isEnabled": "true", "isAuditEnabled": true, 'tableType': 'Inclusion', 'columnType': 'Inclusion', 'policyItems': [ "accesses": [ 'type': 'read' 'isAllowed': true , 'type': 'write' 'isAllowed': true , 'type': 'create' 'isAllowed': true , 'type': 'admin' 'isAllowed': true ], 'users': ["#{options.user.name}"] 'groups': [] 'conditions': [] 'delegateAdmin': true ] ### Indexing Engine Atlas support only solr on cloud mode. Atlas' Ryba installation support solrcoud in or out of docker. options.indexing_engine ?= 'solr' options.solr_client_source ?= service.deps.solr_client.options.source options.solr_client_source = if options.solr_client_source is 'HDP' then '/opt/lucidworks-hdpsearch/solr' else '/usr/solr/current' if options.indexing_engine is 'solr' options.solr ?= {} options.solr_type ?= 'cloud' switch options.solr_type when 'external' options.solr.cluster_config ?= {} options.solr.cluster_config.atlas_collection_dir ?= '/tmp/atlas-infra' options.solr.cluster_config.authentication ?= service.deps.hadoop_core.options.core_site['hadoop.security.authentication'] throw Error "Missing Solr options.solr.cluster_config.user property example: solr" unless options.solr.cluster_config.user? throw Error "Missing Solr options.solr.cluster_config.ssl_enabled property example: true" unless options.solr.cluster_config.ssl_enabled? throw Error "Missing Solr options.solr.cluster_config.hosts: ['master01.ryba', 'master02.ryba']" unless options.solr.cluster_config.hosts? throw Error "Missing Solr options.solr.cluster_config.zk_urls: master01.metal.ryba:2181" unless options.solr.cluster_config.zk_urls? throw Error "Missing Solr options.solr.cluster_config.zk_node: /solr_infra" unless options.solr.cluster_config.zk_node? throw Error "Missing Solr options.solr.cluster_config.master: master01.metal.ryba" unless options.solr.cluster_config.master? throw Error "Missing Solr options.solr.cluster_config.port: 8983" unless options.solr.cluster_config.port? throw Error "Missing Solr options.solr.cluster_config.admin_principal: 8983" unless options.solr.cluster_config.admin_principal? throw Error "Missing Solr options.solr.cluster_config.admin_password: 8983" unless options.solr.cluster_config.admin_password? throw Error "Missing Solr options.solr.cluster_config.authentication: kerberos" unless options.solr.cluster_config.authentication? options.solr.cluster_config.zk_connect ?= path.join options.solr.cluster_config.zk_urls, '/', options.solr.cluster_config.zk_node options.application.properties['atlas.graph.index.search.solr.zookeeper-url'] ?= options.solr.cluster_config.zk_connect options.application.properties['atlas.graph.index.search.solr.mode'] ?= 'cloud' options.application.properties['atlas.graph.index.search.backend'] ?= 'solr5' when 'cloud' throw Error 'No Solr Cloud Server configured' unless service.deps.solr_cloud.length > 0 # options.solr_admin_user ?= 'solr' # options.solr_admin_password ?= 'SolrRocks' #Default options.solr.ssl = options.solr_cloud.options.ssl options.solr.cluster_config ?= user: options.solr_cloud.options.user atlas_collection_dir: "#{options.user.home}/atlas-infra" hosts: service.deps.solr_cloud.map (srv) -> srv.node.fqdn zk_urls: options.solr_cloud.options.zkhosts zk_connect: options.solr_cloud.options.zk_connect master: service.deps.solr_cloud[0].node.fqdn port: options.solr_cloud.options.port authentication: service.deps.hadoop_core.options.core_site['hadoop.security.authentication'] ssl_enabled: options.solr_cloud.options.ssl.enabled if service.deps.hadoop_core.options.core_site['hadoop.security.authentication'] is 'kerberos' options.solr.cluster_config.admin_principal = options.solr_cloud.options.admin_principal options.solr.cluster_config.admin_password = options.solr_cloud.options.admin_password urls = options.solr_cloud.options.zk_connect.split(',').map( (host) -> "#{host}/#{options.solr_cloud.options.zk_node}").join(',') options.application.properties['atlas.graph.index.search.solr.zookeeper-url'] ?= urls options.application.properties['atlas.graph.index.search.solr.mode'] ?= 'cloud' options.application.properties['atlas.graph.index.search.backend'] ?= 'solr5' # # register collection creation just once # if @contexts('ryba/atlas').map( (ctx) -> ctx.config.host )[0] is service.node.fqdn # sc_ctxs[0] # .after # action: ['service','start'] # name: 'solr' # , -> @call 'ryba/atlas/solr_layout' # options.solr_type ?= 'cloud_docker' # solr_ctx = {} # options.application.properties['atlas.solr.kerberos.enable'] ?= if service.deps.hadoop_core.options.core_site['hadoop.security.authentication'] is 'kerberos' then 'true' else 'false' # switch options.solr_type # when 'cloud' # throw Error 'No Solr Cloud Server configured' unless sc_ctxs.length > 0 # options.solr_admin_user ?= 'solr' # options.solr_admin_password ?= 'SolrRocks' #Default # options.cluster_config ?= # atlas_collection_dir: "#{sc_ctxs[0].config.ryba.solr.cloud.conf_dir}/clusters/atlas_solr" # hosts: sc_ctxs.map( (ctx)-> ctx.config.host) # # register collection creation just once # if @contexts('ryba/atlas').map( (ctx) -> ctx.config.host )[0] is service.node.fqdn # sc_ctxs[0] # .after # action: ['service','start'] # name: 'solr' # , -> @call 'ryba/atlas/solr_layout' # when 'cloud_docker' # throw Error 'No Solr Cloud Server configured' unless service.deps.solr_cloud_docker.length > 0 # options.solr_cluster_name ?= 'atlas_infra' # options.solr_admin_principal ?= 'solr' # throw Error 'missing Solr Admin Principal for titan database' unless options.solr_admin_password? # options.solr_users ?= [ # name: 'ranger' # secret: 'ranger123' # ] # options.solr ?= merge {}, service.deps.solr_cloud_docker[0].options, options.solr # #configure atlas'titan solr cluster and pass config to solr's context # options.solr.cluster_config ?= {} # options.solr.cluster_config.authentication ?= service.deps.hadoop_core.options.core_site['hadoop.security.authentication'] # if options.solr.cluster_config.authentication is 'kerberos' # options.solr.cluster_config.admin_principal = options.solr_admin_principal # options.solr.cluster_config.admin_password = options.solr_admin_password # options.solr.cluster_config.volumes ?= [] # options.solr.cluster_config.atlas_collection_dir = "#{options.solr.conf_dir}/clusters/#{options.solr_cluster_name}/atlas_solr" # options.solr.cluster_config.volumes.push "#{options.solr.cluster_config.atlas_collection_dir}:/atlas_solr" if options.solr.cluster_config.volumes.indexOf("#{options.solr.conf_dir}/clusters/#{options.solr_cluster_name}/atlas_solr:/atlas_solr") is -1 # options.solr.cluster_config['containers'] ?= options.solr.hosts.length # options.solr.cluster_config['master'] ?= service.deps.solr_cloud_docker[0].node.fqdn # options.solr.cluster_config['heap_size'] ?= '256m' # options.solr.cluster_config['port'] ?= 8985 # options.solr.cluster_config.zk_opts ?= {} # options.solr.cluster_config['hosts'] ?= options.solr.hosts # #Search for a cloud_docker cluster find in solr.cloud_docker.clusters # for srv in service.deps.solr_cloud_docker # srv.options.clusters[options.solr_cluster_name] = configure_solr_cluster options.solr , options.solr_cluster_name, options.solr.cluster_config # options.solr.cluster_config = merge {}, service.deps.solr_cloud_docker[0].options.clusters[options.solr_cluster_name], options.solr.cluster_config # #Concifugre collections # options.solr.collections ?= # 'vertex_index': # 'name': 'vertex_index' # 'numShards': options.solr.cluster_config['hosts'].length # 'replicationFactor': options.solr.cluster_config['hosts'].length-1 # 'maxShardsPerNode': options.solr.cluster_config['hosts'].length # 'collection.configName': 'vertex_index' # 'edge_index': # 'name': 'edge_index' # 'numShards': options.solr.cluster_config['hosts'].length # 'replicationFactor': options.solr.cluster_config['hosts'].length-1 # 'maxShardsPerNode': options.solr.cluster_config['hosts'].length # 'collection.configName': 'edge_index' # 'fulltext_index': # 'name': 'fulltext_index' # 'numShards': options.solr.cluster_config['hosts'].length # 'replicationFactor': options.solr.cluster_config['hosts'].length-1 # 'maxShardsPerNode': options.solr.cluster_config['hosts'].length # 'collection.configName': 'fulltext_index' # console.log 'Warning not forget solr properties ryba/atlas' # urls = cluster_config.zk_connect.split(',').map( (host) -> "#{host}/#{cluster_config.zk_node}").join(',') # options.solr_policy = # service: options.ranger_solr_install['REPOSITORY_NAME'] # name: "atlas-metadata-server-#{options.fqdn}" # description: "Atlas MetaData Server ACL" # isAuditEnabled: true # resources: # collection: # values: [ # 'vertex_index' # 'edge_index' # 'fulltext_index' # ] # policyItems: [{ # "accesses": [ # 'type': 'query' # 'isAllowed': true # , # 'type': 'update' # 'isAllowed': true # , # 'type': 'others' # 'isAllowed': true # , # 'type': 'solr_admin' # 'isAllowed': true # ], # 'users': ["#{options.user.name}"] # 'groups': [] # 'conditions': [] # 'delegateAdmin': true # }] options.solr.collections ?= 'vertex_index': 'name': 'vertex_index' 'numShards': options.solr.cluster_config['hosts'].length 'replicationFactor': options.solr.cluster_config['hosts'].length-1 'maxShardsPerNode': options.solr.cluster_config['hosts'].length 'collection.configName': 'vertex_index' 'edge_index': 'name': 'edge_index' 'numShards': options.solr.cluster_config['hosts'].length 'replicationFactor': options.solr.cluster_config['hosts'].length-1 'maxShardsPerNode': options.solr.cluster_config['hosts'].length 'collection.configName': 'edge_index' 'fulltext_index': 'name': 'fulltext_index' 'numShards': options.solr.cluster_config['hosts'].length 'replicationFactor': options.solr.cluster_config['hosts'].length-1 'maxShardsPerNode': options.solr.cluster_config['hosts'].length 'collection.configName': 'fulltext_index' ## Wait options.wait ?= {} options.wait.tcp ?= for srv in service.deps.atlas host: srv.node.fqdn port: srv.options.port or 21443 options.wait_solr ?= switch options.solr_type when 'cloud_docker' then options.solr.cluster_config['hosts'].map (host) -> host: host, port: options.solr.cluster_config['port'] options.wait_kafka = service.deps.kafka_broker[0].options.wait options.wait_ranger = service.deps.ranger_admin.options.wait options.wait_hbase = service.deps.hbase_master[0].options.wait options.wait_krb5_client = service.deps.krb5_client.options.wait options.wait_zookeeper_server = service.deps.zookeeper_server[0].options.wait options.wait ?= {} options.wait.http = for srv in service.deps.atlas srv.options.application.properties['atlas.enableTLS'] ?= options.application.properties['atlas.enableTLS'] srv.options.application.properties["atlas.server.http.port"] ?= '21000' srv.options.application.properties["atlas.server.https.port"] ?= '21443' host: srv.node.fqdn port: unless srv.options.application.properties['atlas.enableTLS'] is 'true' then srv.options.application.properties["atlas.server.http.port"] else srv.options.application.properties["atlas.server.https.port"] ## Utility function add_prop = (value, add, separator) -> throw Error 'No separator provided' unless separator? value ?= '' return add if value.length is 0 return if value.indexOf(add) is -1 then "#{value}#{separator}#{add}" else value ## Dependencies configure_solr_cluster = require '../solr/cloud_docker/clusterize' {merge} = require '@nikitajs/core/lib/misc' path = require 'path' [titan]:(http://titan.thinkaurelius.com)