Coverage for sherlock/commonutils/update_wiki

Hot-keys on this page

r m x p toggle line displays

j k next/prev highlighted chunk

0 (zero) top of page

1 (one) first highlighted chunk

1#!/usr/local/bin/python

2# encoding: utf-8

3"""

4*Update sherlock's github wiki pages with some useful info regarding the crossmatch database catalogue tables*

6:Author:

7 David Young

8"""

9from __future__ import print_function

10from builtins import str

11from builtins import object

12import sys

13import os

14os.environ['TERM'] = 'vt100'

15import readline

16import glob

17import collections

18import codecs

19from datetime import datetime, date, time

20import pickle

21from docopt import docopt

22from fundamentals.mysql import readquery

24class update_wiki_pages(object):

25 """

26 *Update sherlock's github wiki pages with some useful info regarding the crossmatch database catalogue tables*

28 **Key Arguments**

30 - ``log`` -- logger

31 - ``settings`` -- the settings dictionary

34 **Usage**

36 To trigger an update of sherlock's wiki pages to give an overview of the crossmatch table database tables run the following:

38 ```python

39 from sherlock.commonutils import update_wiki_pages

40 wiki = update_wiki_pages(

41 log=log,

42 settings=settings

43 )

44 wiki.update()

45 ```

48 .. todo ::

50 - create a new script for updating sherlock wiki with the snippet above, remove wiki command from cl-utils and add stand alone scripts to the sherlock repo (cleans up the usage and docs for sherlock)

51 - harvest text from wiki pages and then delete them: https://github.com/thespacedoctor/sherlock/wiki

52 """

53 # INITIALISATION

55 def __init__(

56 self,

57 log,

58 settings=False,

60 ):

61 self.log = log

62 log.debug("instansiating a new 'update_wiki_pages' object")

63 self.settings = settings

65 # INITIAL ACTIONS

66 # SETUP ALL DATABASE CONNECTIONS

67 from sherlock import database

68 db = database(

69 log=self.log,

70 settings=self.settings

71 )

72 dbConns, dbVersions = db.connect()

73 self.transientsDbConn = dbConns["transients"]

74 self.cataloguesDbConn = dbConns["catalogues"]

76 self.basicColumns = [

77 "view_name",

78 "master table",

79 "description",

80 "version_number",

81 "object_types",

82 "object_type",

83 "number_of_rows",

84 "url",

85 "object_type_accuracy",

86 "last_updated",

87 ]

89 return None

91 def update(self):

92 """

93 Update wiki pages

95 See class docstring for usage

96 """

97 self.log.debug('starting the ``update`` method')

99 if "sherlock wiki root" not in self.settings:

100 print("Sherlock wiki settings not found in settings file")

101 return

102 else:

103 from os.path import expanduser

104 home = expanduser("~")

105 self.settings["sherlock wiki root"] = self.settings[

106 "sherlock wiki root"].replace("~", home)

107

108 staticTableInfo = self._get_table_infos()

109 viewInfo = self._get_view_infos()

110 streamedTableInfo = self._get_stream_view_infos()

111 self._create_md_tables(

112 tableData=staticTableInfo,

113 viewData=viewInfo,

114 streamData=streamedTableInfo

115 )

116 self._write_wiki_pages()

117 self._update_github()

118

119 self.log.debug('completed the ``update`` method')

120 return

121

122 def _get_table_infos(

123 self,

124 trimmed=False):

125 """query the sherlock-catalogues database table metadata

126 """

127 self.log.debug('starting the ``_get_table_infos`` method')

128

129 sqlQuery = u"""

130 SELECT * FROM tcs_helper_catalogue_tables_info where legacy_table = 0 and table_name not like "legacy%%" and table_name not like "%%stream" order by number_of_rows desc;

131 """ % locals()

132 tableInfo = readquery(

133 log=self.log,

134 sqlQuery=sqlQuery,

135 dbConn=self.cataloguesDbConn,

136 quiet=False

137 )

138

139 if trimmed:

140 cleanTable = []

141 for r in tableInfo:

142 orow = collections.OrderedDict(sorted({}.items()))

143 for c in self.basicColumns:

144 if c in r:

145 orow[c] = r[c]

146 cleanTable.append(orow)

147 tableInfo = cleanTable

148

149 self.log.debug('completed the ``_get_table_infos`` method')

150 return tableInfo

151

152 def _get_view_infos(

153 self,

154 trimmed=False):

155 """query the sherlock-catalogues database view metadata

156 """

157 self.log.debug('starting the ``_get_view_infos`` method')

158

159 sqlQuery = u"""

160 SELECT v.*, t.description as "master table" FROM tcs_helper_catalogue_views_info as v, tcs_helper_catalogue_tables_info AS t where v.legacy_view = 0 and v.view_name not like "legacy%%" and t.id=v.table_id order by number_of_rows desc

161 """ % locals()

162 viewInfo = readquery(

163 log=self.log,

164 sqlQuery=sqlQuery,

165 dbConn=self.cataloguesDbConn,

166 quiet=False

167 )

168

169 if trimmed:

170 cleanTable = []

171 for r in viewInfo:

172 orow = collections.OrderedDict(sorted({}.items()))

173 for c in self.basicColumns:

174 if c in r:

175 orow[c] = r[c]

176 cleanTable.append(orow)

177 viewInfo = cleanTable

178

179 self.log.debug('completed the ``_get_view_infos`` method')

180 return viewInfo

181

182 def _get_stream_view_infos(

183 self,

184 trimmed=False):

185 """query the sherlock-catalogues database streamed data tables' metadata

186 """

187 self.log.debug('starting the ``_get_stream_view_infos`` method')

188

189 sqlQuery = u"""

190 SELECT * FROM tcs_helper_catalogue_tables_info where legacy_table = 0 and table_name not like "legacy%%" and table_name like "%%stream" order by number_of_rows desc;

191 """ % locals()

192 streamInfo = readquery(

193 log=self.log,

194 sqlQuery=sqlQuery,

195 dbConn=self.cataloguesDbConn,

196 quiet=False

197 )

198

199 if trimmed:

200 cleanTable = []

201 for r in streamInfo:

202 orow = collections.OrderedDict(sorted({}.items()))