Coverage for gwcelery/conf/__init_

1"""GWCelery application configuration.

3This module defines configuration variables and default values, including both

4:doc:`generic options for Celery <celery:userguide/configuration>` as well as

5options that control the behavior of specific GWCelery :mod:`~gwcelery.tasks`.

7To override the configuration, define the ``CELERY_CONFIG_MODULE`` environment

8variable to the fully qualified name of any Python module that can be located

9in :obj:`sys.path`, including any of the following presets:

11 * :mod:`gwcelery.conf.playground`

12 * :mod:`gwcelery.conf.production`

13 * :mod:`gwcelery.conf.test`

14 * :mod:`gwcelery.conf.dev` (the default)

15"""

17import getpass

18import os

20from igwn_alert.client import DEFAULT_SERVER as DEFAULT_IGWN_ALERT_SERVER

22# Celery application settings.

24# Task tombstones expire after 2 hours.

25# Celery's default setting of 1 day could cause the Redis database to grow too

26# large because we pass large byte strings as task arguments and return values.

27result_expires = 7200

29# Use pickle serializer, because it supports byte values.

30accept_content = ['json', 'pickle']

31event_serializer = 'json'

32result_serializer = 'pickle'

33task_serializer = 'pickle'

35# Compress tasks to reduce bandwidth in and out of Redis.

36result_compression = task_compression = 'zstandard'

38# Task priority settings.

39task_inherit_parent_priority = True

40task_default_priority = 0

41task_queue_max_priority = 1

42broker_transport_options = {

43 'priority_steps': list(range(task_queue_max_priority + 1))

44}

46worker_proc_alive_timeout = 8

47"""The timeout when waiting for a new worker process to start up."""

49worker_log_format = "[%(asctime)s: %(levelname)s/%(processName)s/%(threadName)s] %(message)s" # noqa: E501

50"""Custom worker log format that includes the thread name."""

52# GWCelery-specific settings.

54condor_accounting_group = 'ligo.dev.o4.cbc.pe.bayestar'

55"""HTCondor accounting group for Celery workers launched with condor_submit."""

57expose_to_public = False

58"""Set to True if events meeting the public alert threshold really should be

59exposed to the public."""

61igwn_alert_server = DEFAULT_IGWN_ALERT_SERVER

62"""IGWN alert server"""

64igwn_alert_noauth = False

65"""IGWN alert server no-authetication"""

67igwn_alert_group = 'gracedb-test01'

68"""IGWN alert group."""

70gracedb_host = 'gracedb-test01.igwn.org'

71"""GraceDB host."""

73gwskynet_upper_far_threshold = 1 / 21600

74"""GWSkyNet will only annotate events with FARs below this value."""

76gwskynet_snr_threshold = 4.5

77"""GWSkyNet will only annotate events with individual SNRs above this value."""

79gwskynet_network_snr_threshold = 7.0

80"""GWSkyNet will only annotate events with network SNRs above this value."""

82create_mattermost_channel = False

83"""Do not create Mattermost channel."""

85kafka_consumer_config = {

86 'fermi_gbm_alert': {'url': 'kafka://kafka.gcn.nasa.gov/'

87 'gcn.classic.voevent.FERMI_GBM_ALERT',

88 'suffix': 'xml'},

89 'fermi_gbm_flt_pos': {'url': 'kafka://kafka.gcn.nasa.gov/'

90 'gcn.classic.voevent.FERMI_GBM_FLT_POS',

91 'suffix': 'xml'},

92 'fermi_gbm_gnd_pos': {'url': 'kafka://kafka.gcn.nasa.gov/'

93 'gcn.classic.voevent.FERMI_GBM_GND_POS',

94 'suffix': 'xml'},

95 'fermi_gbm_fin_pos': {'url': 'kafka://kafka.gcn.nasa.gov/'

96 'gcn.classic.voevent.FERMI_GBM_FIN_POS',

97 'suffix': 'xml'},

98 'fermi_gbm_subthresh': {'url': 'kafka://kafka.gcn.nasa.gov/'

99 'gcn.classic.voevent.FERMI_GBM_SUBTHRESH',

100 'suffix': 'xml'},

101 'swift_bat_grb_pos_ack': {'url': 'kafka://kafka.gcn.nasa.gov/'

102 'gcn.classic.voevent.SWIFT_BAT_GRB_POS_ACK',

103 'suffix': 'xml'},

104 'integral_wakeup': {'url': 'kafka://kafka.gcn.nasa.gov/'

105 'gcn.classic.voevent.INTEGRAL_WAKEUP',

106 'suffix': 'xml'},

107 'integral_refined': {'url': 'kafka://kafka.gcn.nasa.gov/'

108 'gcn.classic.voevent.INTEGRAL_REFINED',

109 'suffix': 'xml'},

110 'integral_offline': {'url': 'kafka://kafka.gcn.nasa.gov/'

111 'gcn.classic.voevent.INTEGRAL_OFFLINE',

112 'suffix': 'xml'},

113 'snews': {'url': 'kafka://kafka.gcn.nasa.gov/gcn.classic.voevent.SNEWS',

114 'suffix': 'xml'},

115 'fermi_targeted': {'url': 'kafka://kafka.test.gcn.nasa.gov/'

116 'fermi.gbm.targeted.private.igwn', 'suffix': 'json'},

117 'swift_targeted': {'url': 'kafka://kafka.gcn.nasa.gov/'

118 'gcn.notices.swift.bat.guano', 'suffix': 'json'}

119}

120"""Kafka consumer configuration details. The keys describe the senders of the

121messages to be consumed. The values are a dictionary of the URL to listen to

122and information about the message serializer. NOTE: We will switch the Swift

123URL to a test topic once the topic is regularly written to."""

124

125views_manual_preferred_event_log_message = 'User {} queued a preferred event' \

126 ' change to {}.'

127"""Log message that is uploaded to GraceDB when a user manually changes the

128preferred event in the dashboard."""

129

130voevent_broadcaster_address = ':5342'

131"""The VOEvent broker will bind to this address to send GCNs.

132This should be a string of the form `host:port`. If `host` is empty,

133then listen on all available interfaces."""

134

135voevent_broadcaster_whitelist = []

136"""List of hosts from which the broker will accept connections.

137If empty, then completely disable the broker's broadcast capability."""

138

139voevent_receiver_address = '68.169.57.253:8099'

140"""The VOEvent listener will connect to this address to receive GCNs. For

141options, see `GCN's list of available VOEvent servers

142<https://gcn.gsfc.nasa.gov/voevent.html#tc2>`_. If this is an empty string,

143then completely disable the GCN listener."""

144

145email_host = 'imap.gmail.com'

146"""IMAP hostname to receive the GCN e-mail notice formats."""

147

148superevent_d_t_start = {'gstlal': 1.0,

149 'spiir': 1.0,

150 'pycbc': 1.0,

151 'mbta': 1.0}

152"""Pipeline based lower extent of superevent segments.

153For cwb and lib this is decided from extra attributes."""

154

155superevent_d_t_end = {'gstlal': 1.0,

156 'spiir': 1.0,

157 'pycbc': 1.0,

158 'mbta': 1.0}

159"""Pipeline based upper extent of superevent segments

160For cwb and lib this is decided from extra attributes."""

161

162superevent_query_d_t_start = 100.

163"""Lower extent of superevents query"""

164

165superevent_query_d_t_end = 100.

166"""Upper extent of superevents query"""

167

168superevent_default_d_t_start = 1.0

169"""Default lower extent of superevent segments"""

170

171superevent_default_d_t_end = 1.0

172"""Default upper extent for superevent segments"""

173

174superevent_far_threshold = 1 / 3600

175"""Maximum false alarm rate to consider events superevents."""

176

177superevent_candidate_preference = {

178 'cbc': {'allsky': 2, 'earlywarning': 1, 'ssm': 1, 'mdc': 1},

179 'burst': {'allsky': 0, 'bbh': 0, 'mdc': 0},

180}

181"""Group/search preference for individual candidates. This is

182used by :meth:`gwcelery.tasks.superevents.keyfunc` to sort

183candidates for the preferred event before a ranking statistic is used."""

184

185significant_alert_far_threshold = {

186 'cbc': {

187 'allsky': 1 / (30 * 86400),

188 'earlywarning': 1 / (30 * 86400),

189 'mdc': 1 / (30 * 86400),

190 'ssm': 1 / (365 * 86400),

191 },

192 'burst': {

193 'allsky': 1 / (365 * 86400),

194 'bbh': 1 / (30 * 86400)

195 },

196 'test': {

197 'allsky': 1 / (30 * 86400),

198 'bbh': 1 / (30 * 86400),

199 'earlywarning': 1 / (30 * 86400),

200 'ssm': 1 / (365 * 86400)

201 }

202}

203"""Group and search specific maximum false alarm rate to consider sending

204significant alerts. A threshold of negative infinity disables alerts."""

205

206significant_alert_trials_factor = {

207 'cbc': {'allsky': 6,

208 'earlywarning': 4,

209 'mdc': 6,

210 'ssm': 2},

211 'burst': {'allsky': 3,

212 'bbh': 6}

213}

214"""Trials factor corresponding to trigger categories. The CBC AllSky and Burst

215BBH searches are treated as one group with a common trials factor. CBC AllSky

216pipelines are gstlal, pycbc, mbta, spiir, and raven. The Burst BBH pipeline

217is cwb. CBC EarlyWarning pipelines are gstlal, pycbc, mbta, and spiir.

218CBC SSM pipelines are gstlal and mbta.

219The Burst AllSky searches are treated as one group with one

220trials factor. The Burst AllSky piplines are cwb, mly, and raven."""

221

222preliminary_alert_trials_factor = {

223 'cbc': {'allsky': 7,

224 'earlywarning': 4,

225 'mdc': 4,

226 'ssm': 2},

227 'burst': {'allsky': 7,

228 'bbh': 7}

229}

230"""Trials factor for less significant alert categories. The CBC AllSky, Burst

231AllSky, and Burst BBH searches are all treated as one group with a shared

232trials factor. CBC AllSky pipelines are gstlal, pycbc, mbta, and spiir.

233Burst AllSky pipelines are cwb, and mly. The Burst BBH pipelines is cwb."""

234

235preliminary_alert_far_threshold = {

236 'cbc': {

237 'allsky': 2 / (1 * 86400) * preliminary_alert_trials_factor['cbc']['allsky'], # noqa: E501

238 'earlywarning': -1 * float('inf'),

239 'mdc': -1 * float('inf'),

240 'ssm': -1 * float('inf')

241 },

242 'burst': {

243 'allsky': 2 / (1 * 86400) * preliminary_alert_trials_factor

244 ['burst']['allsky'],

245 'bbh': 2 / (1 * 86400) * preliminary_alert_trials_factor

246 ['burst']['bbh']

247 },

248 'test': {

249 'allsky': 2 / (1 * 86400) * preliminary_alert_trials_factor

250 ['cbc']['allsky'],

251 'earlywarning': -1 * float('inf'),

252 'ssm': -1 * float('inf')

253 }

254}

255"""Group and search specific maximum false alarm rate to consider sending less

256significant alerts. Trials factors are included here to ensure events are sent

257with the false alarm rate initially listed and removing trials factors are from

258the threshold calculation. A threshold of negative infinity disables alerts."""

259

260snews_gw_far_threshold = 1 / (3600 * 24)

261"""Maximum false alarm rate for a superevent to send out a coincidence alert

262between an external SNEWS alert and the superevent."""

263

264sog_paper_far_threshold = {'gw': 1 / (10 * 365 * 86400),

265 'joint': 1 / (10000 * 365 * 86400)}

266"""False alarm rate thresholds for producing a manuscript of speed of gravity

267measurement in low-latency."""

268

269superevent_clean_up_timeout = 270.

270"""The orchestrator will wait this many seconds from the time of the

271application of the GCN_PRELIM_SENT label to revise the preferred

272event out of the accumulated events."""

273

274pe_timeout = 345.0

275"""The orchestrator will wait this many seconds from the time of the

276creation of a new superevent to the time that parameter estimation begins, in

277case the preferred event is updated with high latency."""

278

279rapidpe_timeout = 30.0

280"""The orchestrator will wait this many seconds from the time of the

281creation of a new superevent to the time rapidpe parameter estimation begins,

282in case the preferred event is updated with high latency."""

283

284

285check_vector_prepost = {'gstlal': [2, 2],

286 'spiir': [2, 2],

287 'pycbc': [2, 2],

288 'MBTA': [2, 2],

289 'oLIB': [1.5, 1.5],

290 'LIB': [1.5, 1.5],

291 'CWB': [1.5, 1.5],

292 'MLy': [1.5, 1.5],

293 'HardwareInjection': [2, 2],

294 'Swift': [2, 2],

295 'Fermi': [2, 2],

296 'INTEGRAL': [2, 2],

297 'SNEWS': [10, 10]}

298"""Seconds before and after the superevent start and end times which the DQ

299vector check will include in its check. Pipeline dependent."""

300

301uses_gatedhoft = {'gstlal': True,

302 'spiir': True,

303 'pycbc': True,

304 'MBTA': True,

305 'oLIB': False,

306 'LIB': False,

307 'CWB': True,

308 'MLy': False,

309 'HardwareInjection': False,

310 'Swift': False,

311 'Fermi': False,

312 'INTEGRAL': False,

313 'SNEWS': False}

314"""Whether or not a pipeline uses gated h(t). Determines whether or not

315the DMT-DQ_VECTOR will be analyzed for data quality."""

316

317llhoft_glob = '/dev/shm/kafka/{detector}_O3ReplayMDC/*.gwf'

318"""File glob for playground low-latency h(t) frames. Currently points

319to O3 MDC Mock Data Challange data.

320See https://git.ligo.org/emfollow/mock-data-challenge"""

321

322llhoft_channels = {

323 'H1:DMT-DQ_VECTOR': 'dmt_dq_vector_bits',

324 'L1:DMT-DQ_VECTOR': 'dmt_dq_vector_bits',

325 'H1:GDS-CALIB_STATE_VECTOR': 'ligo_state_vector_bits',

326 'L1:GDS-CALIB_STATE_VECTOR': 'ligo_state_vector_bits',

327 'V1:DQ_ANALYSIS_STATE_VECTOR': 'virgo_state_vector_bits'}

328"""Low-latency h(t) state vector configuration. This is a dictionary consisting

329of a channel and its bitmask, as defined in :mod:`gwcelery.tasks.detchar`."""

330

331idq_ok_channels = ['H1:IDQ-OK_OVL_10_2048',

332 'L1:IDQ-OK_OVL_10_2048']

333"""Low-latency iDQ OK channel names for O4. High bit indicates iDQ is ok."""

334

335idq_channels = ['H1:IDQ-FAP_OVL_10_2048',

336 'L1:IDQ-FAP_OVL_10_2048']

337"""Low-latency iDQ false alarm probability channel names for O4."""

338

339idq_fap_thresh = 0.01

340"""If FAP is below this threshold, and

341:obj:`~gwcelery.conf.idq_veto` for the pipeline is true, DQV will be labeled

342for the event.

343"""

344

345idq_veto = {'gstlal': False,

346 'spiir': False,

347 'pycbc': False,

348 'MBTA': False,

349 'oLIB': False,

350 'LIB': False,

351 'CWB': False,

352 'MLy': False,

353 'HardwareInjection': False,

354 'Swift': False,

355 'Fermi': False,

356 'INTEGRAL': False,

357 'SNEWS': False}

358"""If true for a pipeline, iDQ values below the threshold defined in

359:obj:`~gwcelery.conf.idq_fap_thresh` will cause DQV to be labeled.

360Currently all False, pending iDQ review (should be done before O3).

361"""

362

363low_latency_frame_types = {'H1': 'H1_O3ReplayMDC_llhoft',

364 'L1': 'L1_O3ReplayMDC_llhoft',

365 'V1': 'V1_O3ReplayMDC_llhoft'}

366"""Types of low latency frames used in Parameter Estimation (see

367:mod:`gwcelery.tasks.inference`) and in cache creation for detchar

368checks (see :mod:`gwcelery.tasks.detchar`).

369"""

370

371high_latency_frame_types = {'H1': None,

372 'L1': None,

373 'V1': None}

374"""Types of high latency frames used in Parameter Estimation and in cache

375creation for detchar checks. They do not exist for O3Replay data. (see

376:mod:`gwcelery.tasks.inference` and :mod:`gwcelery.tasks.detchar`)

377"""

378

379strain_channel_names = {'H1': 'H1:GDS-CALIB_STRAIN_INJ1_O3Replay',

380 'L1': 'L1:GDS-CALIB_STRAIN_INJ1_O3Replay',

381 'V1': 'V1:Hrec_hoft_16384Hz_INJ1_O3Replay'}

382"""Names of h(t) channels used in Parameter Estimation (see

383:mod:`gwcelery.tasks.inference`) and in detchar omegascan creation

384(see :mod:`gwcelery.tasks.detchar`)."""

385

386state_vector_channel_names = {'H1': 'H1:GDS-CALIB_STATE_VECTOR',

387 'L1': 'L1:GDS-CALIB_STATE_VECTOR',

388 'V1': 'V1:DQ_ANALYSIS_STATE_VECTOR'}

389"""Names of state vector channels used in Parameter Estimation (see

390:mod:`gwcelery.tasks.inference`)"""

391

392detchar_bit_definitions = {

393 'dmt_dq_vector_bits': {

394 'channel': 'DMT-DQ_VECTOR',

395 'bits': {

396 1: 'NO_OMC_DCPD_ADC_OVERFLOW',

397 2: 'NO_DMT-ETMY_ESD_DAC_OVERFLOW'

398 },

399 'description': {

400 'NO_OMC_DCPD_ADC_OVERFLOW': 'OMC DCPC ADC not overflowing',

401 'NO_DMT-ETMY_ESD_DAC_OVERFLOW': 'ETMY ESD DAC not overflowing'

402 }

403 },

404 'ligo_state_vector_bits': {

405 'channel': 'GDS-CALIB_STATE_VECTOR',

406 'bits': {

407 0: 'HOFT_OK',

408 1: 'OBSERVATION_INTENT',

409 5: 'NO_STOCH_HW_INJ',

410 6: 'NO_CBC_HW_INJ',

411 7: 'NO_BURST_HW_INJ',

412 8: 'NO_DETCHAR_HW_INJ'

413 },

414 'description': {

415 'HOFT_OK': 'h(t) was successfully computed',

416 'OBSERVATION_INTENT': '"observation intent" button is pushed',

417 'NO_STOCH_HW_INJ': 'No stochastic HW injection',

418 'NO_CBC_HW_INJ': 'No CBC HW injection',

419 'NO_BURST_HW_INJ': 'No burst HW injection',

420 'NO_DETCHAR_HW_INJ': 'No HW injections for detector characterization' # noqa: E501

421 }

422 },

423 'virgo_state_vector_bits': {

424 'channel': 'DQ_ANALYSIS_STATE_VECTOR',

425 'bits': {

426 0: 'HOFT_OK',

427 1: 'OBSERVATION_INTENT',

428 5: 'NO_STOCH_HW_INJ',

429 6: 'NO_CBC_HW_INJ',

430 7: 'NO_BURST_HW_INJ',

431 8: 'NO_DETCHAR_HW_INJ',

432 10: 'GOOD_DATA_QUALITY_CAT1'

433 },

434 'description': {

435 'HOFT_OK': 'h(t) was successfully computed',

436 'OBSERVATION_INTENT': '"observation intent" button is pushed',

437 'NO_STOCH_HW_INJ': 'No stochastic HW injection',

438 'NO_CBC_HW_INJ': 'No CBC HW injection',

439 'NO_BURST_HW_INJ': 'No burst HW injection',

440 'NO_DETCHAR_HW_INJ': 'No HW injections for detector characterization', # noqa: E501

441 'GOOD_DATA_QUALITY_CAT1': 'Good data quality (CAT1 type)'

442 }

443 }

444}

445"""Bit definitions for detchar checks"""

446

447omegascan_durations = [(0.75, 0.25), (1.5, 0.5), (7.5, 2.5)]

448"""Durations for omegascans, before and after t0"""

449

450pe_results_path = os.path.join(os.getenv('HOME'), 'public_html/online_pe')

451"""Path to the results of Parameter Estimation (see

452:mod:`gwcelery.tasks.inference`)"""

453

454pe_results_url = ('https://ldas-jobs.ligo.caltech.edu/~{}/'

455 'online_pe/').format(getpass.getuser())

456"""URL of page where all the results of Parameter Estimation are outputted

457(see :mod:`gwcelery.tasks.inference`)"""

458

459raven_coincidence_windows = {'GRB_CBC': [-5, 1],

460 'GRB_CBC_SubFermi': [-11, 1],

461 'GRB_CBC_SubSwift': [-20, 10],

462 'GRB_Burst': [-600, 60],

463 'SNEWS': [-10, 10]}

464"""Time coincidence windows passed to ligo-raven. External events and

465superevents of the appropriate type are considered to be coincident if

466within time window of each other."""

467

468raven_ext_rates = {

469 'GRB': 305 / (3600 * 24 * 365), # 305 / yr

470 'MDC': 305 / (3600 * 24 * 365), # 305 / yr

471 'SubGRB': 370 / (3600 * 24 * 365) # 370 / yr

472}

473"""Expected rates of astrophysical external triggers submitted for each search.

474These potentially include multiple satellites and experiments (e.g. for GRBs,

475Fermi+Swift+INTEGRAL), accounting for multiple detections of the same

476astrophysical event. For more details on methodology, see:

477https://dcc.ligo.org/T1900297."""

478

479raven_targeted_far_thresholds = {

480 'GW': {

481 'Fermi': preliminary_alert_far_threshold['cbc']['allsky'],

482 'Swift': preliminary_alert_far_threshold['cbc']['allsky']

483 },

484 'GRB': {

485 'Fermi': 1 / 10000,

486 'Swift': 1 / 1000

487 }

488}

489"""Max FAR thresholds used for the subthreshold targeted searches with Fermi

490and Swift. Since we only listen to CBC low significance alerts, we use that

491FAR threshold for now. Note that Swift current listens to events with the

492threshold before and Fermi after trials factors."""

493

494external_search_preference = {

495 'Supernova': 2,

496 'GRB': 1,

497 'FRB': 1,

498 'HEN': 1,

499 'SubGRB': 0,

500 'SubGRBTargeted': 0

501}

502"""Group/search preference for individual candidates. This is

503used by :meth:`gwcelery.tasks.raven.keyfunc` to sort

504candidates for the preferred external event. We first prefer coincidences with

505SNEWS Supernova since their significance is so high (far<1/100 yrs). Next, we

506prefer events significant enough to be publishable on their own, which

507includes gamma-ray bursts, high energy neutrinos, and fast radio bursts.

508Lastly we prefer lower significance subthreshold events, some of which are not

509publishable on their own."""

510

511mock_events_simulate_multiple_uploads = False

512"""If True, then upload each mock event several times in rapid succession with

513random jitter in order to simulate multiple pipeline uploads."""

514

515only_alert_for_mdc = False

516"""If True, then only sends alerts for MDC events. Useful for times outside

517of observing runs."""

518

519joint_mdc_freq = 2

520"""Determines how often an external MDC event will be created near an

521MDC superevent to test the RAVEN alert pipeline, i.e for every x

522MDC superevents an external MDC event is created."""

523

524joint_O3_replay_freq = 10

525"""Determines how often an external replay event will be created near an

526superevent to test the RAVEN alert pipeline, i.e for every x

527O3 replay superevents an external MDC event is created."""

528

529condor_retry_kwargs = dict(

530 max_retries=80, retry_backoff=True, retry_jitter=False,

531 retry_backoff_max=600

532)

533"""Retry settings of condor.submit task. With these settings, a condor job is

534no longer tracked ~12 hours after it starts."""

535

536rapidpe_condor_retry_kwargs = dict(

537 max_retries=125, retry_backoff=True, retry_jitter=False,

538 retry_backoff_max=30

539)

540"""Retry settings of condor.submit_rapidpe task. With these settings, a

541condor job is no longer tracked ~1 hours after it starts. This is used

542for RapidPE."""

543

544rapidpe_settings = {

545 'run_mode': 'online',

546 'accounting_group': 'ligo.dev.o4.cbc.pe.lalinferencerapid',

547 'use_cprofile': True,

548}

549"""Config settings used for rapidpe"""

550

551# Delete imported modules so that they do not pollute the config object

552del os, getpass

Coverage for gwcelery/conf/init.py: 100%

138 statements