ComboCode.cc.managers.ModelingManager

20 21 """ 22 A modeling manager which maintains information for all modeling sessions. 23 24 """ 25

26 - def __init__(self,var_pars,processed_input,iterations=1,\ 27 mcmax=0,gastronoom=0,sphinx=0,iterative=0,\ 28 num_model_sessions=1,vic_manager=None,replace_db_entry=0,\ 29 path_gastronoom='runTest',path_mcmax='runTest',\ 30 skip_cooling=0,recover_sphinxfiles=0,single_session=0):

31 32 """ 33 Initializing a ModelingManager instance. 34 35 From this class, the MCMax and GASTRoNOoM codes are ran for each 36 iteration, as well as the database retrieval of older models. 37 38 @param var_pars: gridded parameters in the CC session 39 @type var_pars: list[string] 40 @param processed_input: The processed paramaters from the CC inputfile 41 @type processed_input: dict 42 @keyword iterations: Number of iterations 43 44 (default: 1) 45 @type iterations: int 46 @keyword mcmax: Running MCMax? 47 48 (default: 0) 49 @type mcmax: bool 50 @keyword gastronoom: Running GASTRoNOoM? 51 52 (default: 0 53 @type gastronoom: bool 54 @keyword sphinx: Running Sphinx? 55 56 (default: 0) 57 @type sphinx: bool 58 @keyword iterative: Ray-trace MCMax models on every iteration 59 60 (default: 0) 61 @type iterative: bool 62 @keyword num_model_sessions: number of sessions, i.e. len(star_grid) 63 64 (default: 1) 65 @type num_model_sessions: int 66 @keyword vic_manager: the vic manager to run models (sphinx) on VIC3 67 68 (default: None) 69 @type vic_manager: Vic() 70 @keyword replace_db_entry: replace an entry in the database with a 71 newly calculated model with a new model id 72 (eg if some general data not included in 73 the inputfiles is changed) 74 75 (default: 0) 76 @type replace_db_entry: bool 77 @keyword skip_cooling: Skip running cooling in case a model is not 78 found in the database, for instance if it is 79 already known that the model will fail 80 81 (default: 0) 82 @type skip_cooling: bool 83 @keyword recover_sphinxfiles: Try to recover sphinx files from the disk 84 in case they were correctly calculated, 85 but not saved to the database for one 86 reason or another. 87 88 (default: 0) 89 @type recover_sphinxfiles: bool 90 @keyword path_mcmax: modeling folder in MCMax home 91 92 (default: 'runTest') 93 @type path_mcmax: string 94 @keyword path_gastronoom: modeling folder in GASTRoNOoM home 95 96 (default: 'runTest') 97 @type path_gastronoom: string 98 @keyword single_session: If this is the only CC session. Speeds up db 99 check. 100 101 (default: 0) 102 @type single_session: bool 103 104 """ 105 106 self.var_pars, self.iterations = var_pars, int(iterations) 107 self.mcmax, self.gastronoom = int(mcmax), int(gastronoom) 108 self.sphinx = int(sphinx) 109 self.skip_cooling = skip_cooling 110 self.input_dict = processed_input 111 self.iterative = int(iterative) 112 self.star_grid_old = [[] for i in xrange(num_model_sessions)] 113 self.vic = vic_manager 114 self.replace_db_entry = replace_db_entry 115 self.new_entries_mcmax = [] 116 self.new_entries_cooling = [] 117 self.trans_bool_list = [] 118 self.mline_done_list = [] 119 self.mcmax_done_list = [] 120 self.path_mcmax = path_mcmax 121 self.path_gastronoom = path_gastronoom 122 self.recover_sphinxfiles = recover_sphinxfiles 123 self.single_session = single_session 124 125 #-- Convenience paths 126 cc.path.gout = os.path.join(cc.path.gastronoom,self.path_gastronoom) 127 cc.path.mout = os.path.join(cc.path.mcmax,self.path_mcmax) 128 self.setDatabases() 129 130 #-- Initialize code booleans for checking if a code was ran 131 self.mcmax_done = False 132 self.mline_done = False

133 134

135 - def setDatabases(self):

136 137 ''' 138 Initialize all databases relevant for this grid. 139 140 ''' 141 142 if self.gastronoom: 143 cool_db_path = os.path.join(cc.path.gout,\ 144 'GASTRoNOoM_cooling_models.db') 145 ml_db_path = os.path.join(cc.path.gout,\ 146 'GASTRoNOoM_mline_models.db') 147 sph_db_path = os.path.join(cc.path.gout,\ 148 'GASTRoNOoM_sphinx_models.db') 149 self.cool_db = Database.Database(db_path=cool_db_path) 150 self.ml_db = Database.Database(db_path=ml_db_path) 151 self.sph_db = Database.Database(db_path=sph_db_path) 152 if self.mcmax: 153 mcmax_db_path = os.path.join(cc.path.mout,'MCMax_models.db') 154 self.mcmax_db = Database.Database(db_path=mcmax_db_path) 155 if not self.vic is None: 156 self.vic.setSphinxDb(self.sph_db)

157 158 159

160 - def startModeling(self,star,star_index):

161 162 """ 163 Start the modeling process on a model star. 164 165 @param star: The parameter set for this session 166 @type star: Star() 167 @param star_index: The index of the Star() object in the full list in 168 CC. Only used to track earlier iterations if 169 iterative==1 170 @type star_index: int 171 172 """ 173 174 #-- Note that db synchronisation is done every time a model is 175 # successfully calculated, every time a previous code has been ran, 176 # and every time a change is made to the database, in terms of model 177 # id reservation. 178 179 for i in range(self.iterations): 180 if self.mcmax: 181 print '***********************************' 182 print '** Starting MCMax calculation.' 183 print '** Iteration # %i for Model %i.'%(i+1,star_index+1) 184 #-- Check if the previously calculated GASTRoNOoM model was 185 # calculated. This means a lot of time has passed since 186 # the last synchronisation of the mcmax db, so sync it 187 #Not needed anymore. Sync happens before db check 188 # if self.mline_done: 189 # self.mcmax_db.sync() 190 #-- Initiate a dust session which is used for every iteration 191 if i == 0: 192 dust_session = MCMax(path_mcmax=self.path_mcmax,\ 193 db=self.mcmax_db,\ 194 new_entries=self.new_entries_mcmax,\ 195 replace_db_entry=self.replace_db_entry,\ 196 single_session=self.single_session) 197 self.mcmax_done = False 198 dust_session.doMCMax(star) 199 if dust_session.mcmax_done: 200 self.mcmax_done = True 201 202 #-- In case a cooling model was in progress, wait until 203 # finished. 204 while dust_session.in_progress: 205 self.mcmax_db.sync() 206 if not self.mcmax_db.has_key(dust_session.model_id): 207 print 'MCMax model calculation failed.' 208 dust_session.model_id = '' 209 break 210 elif not self.mcmax_db[dust_session.model_id]\ 211 .has_key('IN_PROGRESS'): 212 print 'MCMax model calculation finished.' 213 break 214 print 'MCMax still running in another CC session.'+\ 215 'Waiting 1 minute before checking again.' 216 try: 217 time.sleep(60) 218 except KeyboardInterrupt: 219 print 'Ending wait time, continuing with ' + \ 220 'progress check immediately.' 221 dust_session.in_progress = False 222 223 #-- add/change 'LAST_MCMAX_MODEL' entry. Only do this if model 224 # was calculated successfully. In case of a failure, you want 225 # to keep the model id of a previous iteration if available. 226 if dust_session.model_id: 227 star['LAST_MCMAX_MODEL'] = dust_session.model_id 228 229 #-- If last iteration, ray trace. 230 if i+1 == self.iterations or self.iterative: 231 dust_session.rayTrace(star) 232 233 #-- Remember every iteration if iterative==True. 234 if self.iterative: 235 self.star_grid_old[star_index].append(star.copy()) 236 237 #-- add/change MUTABLE input keys, which MAY be overwritten by 238 # the input file inputComboCode.dat 239 # Only relevant if a model match was found/calculated. 240 if dust_session.model_id: 241 star.removeMutableMCMax(dust_session.mutable,self.var_pars) 242 star.update(self.input_dict) 243 if self.replace_db_entry: 244 self.new_entries_mcmax.append(dust_session.model_id) 245 246 if self.gastronoom: 247 #- Initiate a gas session which is used for every iteration 248 if i == 0: 249 gas_session = Gastronoom(vic=self.vic,\ 250 path_gastronoom=self.path_gastronoom,\ 251 cool_db=self.cool_db,\ 252 ml_db=self.ml_db,\ 253 sph_db=self.sph_db,\ 254 sphinx=self.sphinx,\ 255 skip_cooling=self.skip_cooling,\ 256 replace_db_entry=self.replace_db_entry,\ 257 new_entries=self.new_entries_cooling,\ 258 recover_sphinxfiles=self.recover_sphinxfiles,\ 259 single_session=self.single_session) 260 self.mline_done = False 261 #if self.mcmax_done: 262 #-- MCMax was ran successfully, in other words, quite a bit 263 # of time has passed, so update the cool database. 264 #self.cool_db.sync() 265 #Not needed anymore. Sync happens before db check 266 267 #-- If last iteration and no mline is requested, don't run 268 # cooling. There is no point anyway (and means you don't have 269 # to wait for your MCMax model. 270 if (i+1 == self.iterations) and not star['GAS_LIST']: 271 continue 272 273 print '***********************************' 274 print '** Starting GASTRoNOoM calculation.' 275 print '** Iteration # %i for Model %i.'%(i+1,star_index+1) 276 277 #-- Otherwise, run cooling and do the rest of the loop 278 gas_session.doGastronoom(star) 279 280 #-- In case a cooling model was in progress, wait until 281 # finished. 282 while gas_session.in_progress: 283 self.cool_db.sync() 284 if not self.cool_db.has_key(gas_session.model_id): 285 print 'Cooling model calculation failed.' 286 gas_session.model_id = '' 287 break 288 elif not self.cool_db[gas_session.model_id]\ 289 .has_key('IN_PROGRESS'): 290 print 'Cooling model calculation finished.' 291 break 292 print 'Cooling still running in another CC session.'+\ 293 'Waiting 1 minute before checking again.' 294 try: 295 time.sleep(60) 296 except KeyboardInterrupt: 297 print 'Ending wait time, continuing with ' + \ 298 'progress check immediately.' 299 gas_session.in_progress = False 300 301 #-- add/change 'LAST_GASTRONOOM_MODEL' entry. Only do this if 302 # model was calculated successfully. In case of a failure, you 303 # wantto keep the model id of a previous iteration if 304 # available. 305 if gas_session.model_id: 306 star['LAST_GASTRONOOM_MODEL'] = gas_session.model_id 307 308 #- add/change MUTABLE input keys, which MAY be overwritten by 309 #- the input file inputComboCode.dat 310 star.removeMutableGastronoom(gas_session.mutable,self.var_pars) 311 star.update(self.input_dict) 312 star.updateMolecules(parlist=gas_session.mutable) 313 314 #-- Remember the new model_ids if the replace_db_entry option 315 # is on. 316 if self.replace_db_entry: 317 self.new_entries_cooling.append(gas_session.model_id) 318 319 #-- If last iteration, run mline and sphinx. Note that the 320 # model_id cannot have changed: it's either the original 321 # cooling model_id or no model_id at all in case of total fail 322 if (i+1 == self.iterations) and gas_session.model_id: 323 #-- Only sync db if a cooling model was calced just before. 324 # Not needed anymore sync happens before db check. 325 #if gas_session.cool_done: self.ml_db.sync() 326 gas_session.doMline(star) 327 if gas_session.mline_done: self.mline_done = True 328 329 #-- Now check if molecules still in progress have finished 330 while gas_session.molec_in_progress: 331 self.ml_db.sync() 332 still_in_progress = [] 333 for molec in gas_session.molec_in_progress: 334 cid = gas_session.model_id 335 mid = molec.getModelId() 336 md = self.ml_db[cid] 337 mstr = molec.molecule 338 #-- Check if molec id or molecule itself exists in 339 # db. Id may also have been removed, in case all 340 # molecules failed. 341 if not md.has_key(mid) or not md[mid].has_key(mstr): 342 print 'Mline model calculation failed.' 343 molec.setModelId('') 344 elif not md[mid][mstr].has_key('IN_PROGRESS'): 345 print 'Mline model calculation finished for '+\ 346 '%s.'%mstr 347 else: 348 print 'Mline still running in another CC '+\ 349 'session for %s. '%mstr 350 still_in_progress.append(molec) 351 if not still_in_progress: 352 mls = set([molec.getModelId() 353 for molec in gas_session.molec_list]) 354 if mls == set(['']): 355 #- no mline models calculated: stop GASTRoNOoM 356 gas_session.model_id = '' 357 print 'Mline model calculation failed for ' + \ 358 'all requested molecules. Stopping ' + \ 359 'GASTRoNOoM here!' 360 break 361 gas_session.molec_in_progress = still_in_progress 362 print 'Waiting 1 minute before checking again.' 363 try: 364 time.sleep(60) 365 except KeyboardInterrupt: 366 print 'Ending wait time, continuing with ' + \ 367 'progress check immediately.' 368 369 #-- Check if the model id is still valid after the mline run 370 if gas_session.model_id: 371 #-- Only sync the db if an mline model was calculated 372 # just before. Sync now always happens before db check 373 #if self.mline_done: self.sph_db.sync() 374 gas_session.doSphinx(star) 375 print '***********************************' 376 377 #- remember trans bools if sphinx is enabled, so you can trace which 378 #- models have been calculated in this session and which were retrieved 379 #- from database. Also remember if mline was required to be ran this 380 #- session, or rather if everything was taken out of the db 381 if self.gastronoom: 382 self.mline_done_list.append(self.mline_done) 383 if self.sphinx: 384 self.trans_bool_list.append(gas_session.trans_bools) 385 386 #- remember if mcmax was ran, or rather the model taken from the db. 387 #- In first or second iteration, doesn't matter. 388 if self.mcmax: 389 self.mcmax_done_list.append(self.mcmax_done)

Source Code for Module ComboCode.cc.managers.ModelingManager