Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#!/usr/bin/env python
2# encoding: utf-8
3"""
4Documentation for sherlock can be found here: http://sherlock.readthedocs.org
6.. todo ::
8 - docuument cl_utils module
9 - tidy usage text
11Usage:
12 sherlock init
13 sherlock info [-s <pathToSettingsFile>]
14 sherlock [-NA] dbmatch [--update] [-s <pathToSettingsFile>]
15 sherlock [-bN] match -- <ra> <dec> [<pathToSettingsFile>]
16 sherlock clean [-s <pathToSettingsFile>]
17 sherlock wiki [-s <pathToSettingsFile>]
18 sherlock import ned <ra> <dec> <radiusArcsec> [-s <pathToSettingsFile>]
19 sherlock import cat <cat_name> <pathToDataFile> <cat_version> [-s <pathToSettingsFile>]
20 sherlock import stream <stream_name> [-s <pathToSettingsFile>]
22Options:
23 init setup the sherlock settings file for the first time
24 match XXXX
25 dbmatch database match
26 clean XXXX
27 wiki XXXX
28 import XXXX
29 ned use the online NED database as the source catalogue
30 cat import a static catalogue into the sherlock-catalogues database
31 stream download/stream new data from a give source catalogue into the sherlock sherlock-catalogues database
32 info print an overview of the current catalogues, views and streams in the sherlock database ready for crossmatching
34 ra the right-ascension coordinate with which to perform a conesearch (sexegesimal or decimal degrees)
35 dec the declination coordinate with which to perform a conesearch (sexegesimal or decimal degrees)
36 radiusArcsec radius in arcsec of the footprint to download from the online NED database
37 cat_name name of the catalogue being imported (veron|ned_d)
38 stream_name name of the stream to import into the sherlock-catalogues database (ifs)
40 -N, --skipNedUpdate do not update the NED database before classification
41 -A, --skipMagUpdate do not update the peak magnitudes and human readable text annotations of objects (can eat up some time)
42 -h, --help show this help message
43 -s, --settings the settings file
44 -b, --verbose print more details to stdout
45 -u, --update update the transient database with new classifications and crossmatches
46 -v, --version print the version of sherlock
47"""
48from __future__ import print_function
49from __future__ import absolute_import
50import sys
51import os
52os.environ['TERM'] = 'vt100'
53import readline
54import glob
55import pickle
56from docopt import docopt
57from fundamentals import tools, times
58from subprocess import Popen, PIPE, STDOUT
59from fundamentals.renderer import list_of_dictionaries
60from .database_cleaner import database_cleaner
61from .commonutils import update_wiki_pages
62from sherlock.imports import veron as veronImporter
63from sherlock.imports import ifs as ifsImporter
64from sherlock.imports import ned_d as nedImporter
65from sherlock.imports import ned as nedStreamImporter
66from sherlock.commonutils import update_wiki_pages
67from sherlock import transient_classifier
70def tab_complete(text, state):
71 return (glob.glob(text + '*') + [None])[state]
74def main(arguments=None):
75 """
76 *The main function used when `cl_utils.py` is run as a single script from the cl, or when installed as a cl command*
77 """
78 # setup the command-line util settings
79 su = tools(
80 arguments=arguments,
81 docString=__doc__,
82 logLevel="WARNING",
83 options_first=False,
84 distributionName="qub-sherlock",
85 projectName="sherlock",
86 defaultSettingsFile=True
87 )
88 arguments, settings, log, dbConn = su.setup()
90 # tab completion for raw_input
91 readline.set_completer_delims(' \t\n;')
92 readline.parse_and_bind("tab: complete")
93 readline.set_completer(tab_complete)
95 # UNPACK REMAINING CL ARGUMENTS USING `EXEC` TO SETUP THE VARIABLE NAMES
96 # AUTOMATICALLY
97 a = {}
98 for arg, val in list(arguments.items()):
99 if arg[0] == "-":
100 varname = arg.replace("-", "") + "Flag"
101 else:
102 varname = arg.replace("<", "").replace(">", "")
103 a[varname] = val
104 if arg == "--dbConn":
105 dbConn = val
106 a["dbConn"] = val
107 log.debug('%s = %s' % (varname, val,))
109 ## START LOGGING ##
110 startTime = times.get_now_sql_datetime()
111 log.info(
112 '--- STARTING TO RUN THE cl_utils.py AT %s' %
113 (startTime,))
115 # set options interactively if user requests
116 if "interactiveFlag" in a and a["interactiveFlag"]:
118 # load previous settings
119 moduleDirectory = os.path.dirname(__file__) + "/resources"
120 pathToPickleFile = "%(moduleDirectory)s/previousSettings.p" % locals()
121 try:
122 with open(pathToPickleFile):
123 pass
124 previousSettingsExist = True
125 except:
126 previousSettingsExist = False
127 previousSettings = {}
128 if previousSettingsExist:
129 previousSettings = pickle.load(open(pathToPickleFile, "rb"))
131 # x-raw-input
132 # x-boolean-raw-input
133 # x-raw-input-with-default-value-from-previous-settings
135 # save the most recently used requests
136 pickleMeObjects = []
137 pickleMe = {}
138 theseLocals = locals()
139 for k in pickleMeObjects:
140 pickleMe[k] = theseLocals[k]
141 pickle.dump(pickleMe, open(pathToPickleFile, "wb"))
143 if a["init"]:
144 from os.path import expanduser
145 home = expanduser("~")
146 filepath = home + "/.config/sherlock/sherlock.yaml"
147 try:
148 cmd = """open %(filepath)s""" % locals()
149 p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
150 except:
151 pass
152 try:
153 cmd = """start %(filepath)s""" % locals()
154 p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True)
155 except:
156 pass
157 return
159 init = a["init"]
160 match = a["match"]
161 dbmatch = a["dbmatch"]
162 clean = a["clean"]
163 wiki = a["wiki"]
164 iimport = a["import"]
165 ned = a["ned"]
166 cat = a["cat"]
167 stream = a["stream"]
168 info = a["info"]
169 ra = a["ra"]
170 dec = a["dec"]
171 radiusArcsec = a["radiusArcsec"]
172 cat_name = a["cat_name"]
173 stream_name = a["stream_name"]
174 skipNedUpdateFlag = a["skipNedUpdateFlag"]
175 skipMagUpdateFlag = a["skipMagUpdateFlag"]
176 settingsFlag = a["settingsFlag"]
177 verboseFlag = a["verboseFlag"]
178 updateFlag = a["updateFlag"]
180 # CALL FUNCTIONS/OBJECTS
181 if match or dbmatch:
182 if verboseFlag:
183 verbose = 2
184 else:
185 verbose = 1
187 if skipNedUpdateFlag:
188 updateNed = False
189 else:
190 updateNed = True
192 if skipMagUpdateFlag:
193 updatePeakMags = False
194 else:
195 updatePeakMags = True
197 classifier = transient_classifier.transient_classifier(
198 log=log,
199 settings=settings,
200 ra=ra,
201 dec=dec,
202 name=False,
203 verbose=verbose,
204 update=updateFlag,
205 updateNed=updateNed,
206 updatePeakMags=updatePeakMags
207 )
208 classifier.classify()
210 if clean:
211 cleaner = database_cleaner(
212 log=log,
213 settings=settings
214 )
215 cleaner.clean()
216 if wiki:
217 updateWiki = update_wiki_pages(
218 log=log,
219 settings=settings
220 )
221 updateWiki.update()
223 if iimport and ned:
224 ned = nedStreamImporter(
225 log=log,
226 settings=settings,
227 coordinateList=["%(ra)s %(dec)s" % locals()],
228 radiusArcsec=radiusArcsec
229 )
230 ned.ingest()
231 if iimport and cat:
233 if cat_name == "veron":
234 catalogue = veronImporter(
235 log=log,
236 settings=settings,
237 pathToDataFile=pathToDataFile,
238 version=cat_version,
239 catalogueName=cat_name
240 )
241 catalogue.ingest()
243 if "ned_d" in cat_name:
244 catalogue = nedImporter(
245 log=log,
246 settings=settings,
247 pathToDataFile=pathToDataFile,
248 version=cat_version,
249 catalogueName=cat_name
250 )
251 catalogue.ingest()
252 if iimport and stream:
253 if "ifs" in stream_name:
254 stream = ifsImporter(
255 log=log,
256 settings=settings
257 )
258 stream.ingest()
259 if not init and not match and not clean and not wiki and not iimport and ra:
261 classifier = transient_classifier.transient_classifier(
262 log=log,
263 settings=settings,
264 ra=ra,
265 dec=dec,
266 name=False,
267 verbose=verboseFlag
268 )
269 classifier.classify()
271 if info:
272 print("sherlock-catalogues")
273 wiki = update_wiki_pages(
274 log=log,
275 settings=settings
276 )
277 table = list(wiki._get_table_infos(trimmed=True))
279 dataSet = list_of_dictionaries(
280 log=log,
281 listOfDictionaries=table
282 )
283 tableData = dataSet.reST(filepath=None)
284 print(tableData)
285 print()
287 print("Crossmatch Streams")
288 table = list(wiki._get_stream_view_infos(trimmed=True))
289 dataSet = list_of_dictionaries(
290 log=log,
291 listOfDictionaries=table
292 )
293 tableData = dataSet.reST(filepath=None)
294 print(tableData)
295 print()
297 print("Views on Catalogues and Streams")
299 table = list(wiki._get_view_infos(trimmed=True))
300 dataSet = list_of_dictionaries(
301 log=log,
302 listOfDictionaries=table
303 )
304 tableData = dataSet.reST(filepath=None)
305 print(tableData)
307 if "dbConn" in locals() and dbConn:
308 dbConn.commit()
309 dbConn.close()
310 ## FINISH LOGGING ##
311 endTime = times.get_now_sql_datetime()
312 runningTime = times.calculate_time_difference(startTime, endTime)
313 log.info('-- FINISHED ATTEMPT TO RUN THE cl_utils.py AT %s (RUNTIME: %s) --' %
314 (endTime, runningTime, ))
316 return
318if __name__ == '__main__':
319 main()