Package Gnumed :: Package pycommon :: Module gmBackendListener |
[frames] | no frames]

# Source Code for Module Gnumed.pycommon.gmBackendListener

      1  """GNUmed database backend listener.
      2
      3  This module implements threaded listening for asynchronuous
      4  notifications from the database backend.
      5  """
      6  #=====================================================================
      7  __version__ = "$Revision: 1.22 $"
      8  __author__ = "H. Herb <hherb@gnumed.net>, K.Hilbert <karsten.hilbert@gmx.net>"
      9
     10  import sys, time, threading, select, logging
     11
     12
     13  if __name__ == '__main__':
     14          sys.path.insert(0, '../../')
     15  from Gnumed.pycommon import gmDispatcher, gmExceptions, gmBorg
     16
     17
     18  _log = logging.getLogger('gm.db')
     19  _log.info(__version__)
     20
     21
     22  static_signals = [
     23          u'db_maintenance_warning',              # warns of impending maintenance and asks for disconnect
     24          u'db_maintenance_disconnect'    # announces a forced disconnect and disconnects
     25  ]
     26  #=====================================================================
     27 -class gmBackendListener(gmBorg.cBorg):
     28
     29 -        def __init__(self, conn=None, poll_interval=3, patient=None):
     30
     31                  try:
     32                          self.already_inited
     33                          return
     34                  except AttributeError:
     35                          pass
     36
     37                  _log.info('starting backend notifications listener thread')
     38
     39                  # the listener thread will regularly try to acquire
     40                  # this lock, when it succeeds it will quit
     41                  self._quit_lock = threading.Lock()
     42                  # take the lock now so it cannot be taken by the worker
     43                  # thread until it is released in shutdown()
     44                  if not self._quit_lock.acquire(0):
     45                          _log.error('cannot acquire thread-quit lock ! aborting')
     46                          raise gmExceptions.ConstructorError, "cannot acquire thread-quit lock"
     47
     48                  self._conn = conn
     49                  self.backend_pid = self._conn.get_backend_pid()
     50                  _log.debug('connection has backend PID [%s]', self.backend_pid)
     51                  self._conn.set_isolation_level(0)               # autocommit mode
     52                  self._cursor = self._conn.cursor()
     53                  try:
     54                          self._conn_fd = self._conn.fileno()
     55                  except AttributeError:
     56                          self._conn_fd = self._cursor.fileno()
     57                  self._conn_lock = threading.Lock()              # lock for access to connection object
     58
     59                  self.curr_patient_pk = None
     60                  if patient is not None:
     61                          if patient.connected:
     62                                  self.curr_patient_pk = patient.ID
     63                  self.__register_interests()
     64
     65                  # check for messages every 'poll_interval' seconds
     66                  self._poll_interval = poll_interval
     67                  self._listener_thread = None
     68                  self.__start_thread()
     69
     70                  self.already_inited = True
     71          #-------------------------------
     72          # public API
     73          #-------------------------------
     74 -        def shutdown(self):
     75                  if self._listener_thread is None:
     76                          self.__shutdown_connection()
     77                          return
     78
     79                  _log.info('stopping backend notifications listener thread')
     80                  self._quit_lock.release()
     81                  try:
     82                          # give the worker thread time to terminate
     83                          self._listener_thread.join(self._poll_interval+2.0)
     84                          try:
     85                                  if self._listener_thread.isAlive():
     86                                          _log.error('listener thread still alive after join()')
     87                                          _log.debug('active threads: %s' % threading.enumerate())
     88                          except:
     89                                  pass
     90                  except:
     91                          print sys.exc_info()
     92
     93                  self._listener_thread = None
     94
     95                  try:
     96                          self.__unregister_patient_notifications()
     97                  except:
     98                          _log.exception('unable to unregister patient notifications')
     99                  try:
    100                          self.__unregister_unspecific_notifications()
    101                  except:
    102                          _log.exception('unable to unregister unspecific notifications')
    103
    104                  self.__shutdown_connection()
    105
    106                  return
    107          #-------------------------------
    108          # event handlers
    109          #-------------------------------
    110 -        def _on_pre_patient_selection(self, *args, **kwargs):
    111                  self.__unregister_patient_notifications()
    112                  self.curr_patient_pk = None
    113          #-------------------------------
    114 -        def _on_post_patient_selection(self, *args, **kwargs):
    115                  self.curr_patient_pk = kwargs['pk_identity']
    116                  self.__register_patient_notifications()
    117          #-------------------------------
    118          # internal helpers
    119          #-------------------------------
    120 -        def __register_interests(self):
    121
    122                  # determine patient-specific notifications
    123                  cmd = u'SELECT DISTINCT ON (signal) signal FROM gm.notifying_tables WHERE carries_identity_pk IS true'
    124                  self._conn_lock.acquire(1)
    125                  try:
    126                          self._cursor.execute(cmd)
    127                  finally:
    128                          self._conn_lock.release()
    129                  rows = self._cursor.fetchall()
    130                  self.patient_specific_notifications = [ '%s_mod_db' % row[0] for row in rows ]
    131                  _log.info('configured patient specific notifications:')
    132                  _log.info('%s' % self.patient_specific_notifications)
    133                  gmDispatcher.known_signals.extend(self.patient_specific_notifications)
    134
    135                  # determine unspecific notifications
    136                  cmd = u'select distinct on (signal) signal from gm.notifying_tables where carries_identity_pk is False'
    137                  self._conn_lock.acquire(1)
    138                  try:
    139                          self._cursor.execute(cmd)
    140                  finally:
    141                          self._conn_lock.release()
    142                  rows = self._cursor.fetchall()
    143                  self.unspecific_notifications = [ '%s_mod_db' % row[0] for row in rows ]
    144                  self.unspecific_notifications.extend(static_signals)
    145                  _log.info('configured unspecific notifications:')
    146                  _log.info('%s' % self.unspecific_notifications)
    147                  gmDispatcher.known_signals.extend(self.unspecific_notifications)
    148
    149                  # listen to patient changes inside the local client
    150                  # so we can re-register patient specific notifications
    151                  gmDispatcher.connect(signal = u'pre_patient_selection', receiver = self._on_pre_patient_selection)
    152                  gmDispatcher.connect(signal = u'post_patient_selection', receiver = self._on_post_patient_selection)
    153
    154                  # do we need to start listening to patient specific
    155                  # notifications right away because we missed an
    156                  # earlier patient activation ?
    157                  self.__register_patient_notifications()
    158
    159                  # listen to unspecific (non-patient related) notifications
    160                  self.__register_unspecific_notifications()
    161          #-------------------------------
    162 -        def __register_patient_notifications(self):
    163                  if self.curr_patient_pk is None:
    164                          return
    165                  for notification in self.patient_specific_notifications:
    166                          notification = '%s:%s' % (notification, self.curr_patient_pk)
    167                          _log.debug('starting to listen for [%s]' % notification)
    168                          cmd = 'LISTEN "%s"' % notification
    169                          self._conn_lock.acquire(1)
    170                          try:
    171                                  self._cursor.execute(cmd)
    172                          finally:
    173                                  self._conn_lock.release()
    174          #-------------------------------
    175 -        def __unregister_patient_notifications(self):
    176                  if self.curr_patient_pk is None:
    177                          return
    178                  for notification in self.patient_specific_notifications:
    179                          notification = '%s:%s' % (notification, self.curr_patient_pk)
    180                          _log.debug('stopping to listen for [%s]' % notification)
    181                          cmd = 'UNLISTEN "%s"' % notification
    182                          self._conn_lock.acquire(1)
    183                          try:
    184                                  self._cursor.execute(cmd)
    185                          finally:
    186                                  self._conn_lock.release()
    187          #-------------------------------
    188 -        def __register_unspecific_notifications(self):
    189                  for sig in self.unspecific_notifications:
    190                          sig = '%s:' % sig
    191                          _log.info('starting to listen for [%s]' % sig)
    192                          cmd = 'LISTEN "%s"' % sig
    193                          self._conn_lock.acquire(1)
    194                          try:
    195                                  self._cursor.execute(cmd)
    196                          finally:
    197                                  self._conn_lock.release()
    198          #-------------------------------
    199 -        def __unregister_unspecific_notifications(self):
    200                  for sig in self.unspecific_notifications:
    201                          sig = '%s:' % sig
    202                          _log.info('stopping to listen for [%s]' % sig)
    203                          cmd = 'UNLISTEN "%s"' % sig
    204                          self._conn_lock.acquire(1)
    205                          try:
    206                                  self._cursor.execute(cmd)
    207                          finally:
    208                                  self._conn_lock.release()
    209          #-------------------------------
    210 -        def __shutdown_connection(self):
    211                  _log.debug('shutting down connection with backend PID [%s]', self.backend_pid)
    212                  self._conn_lock.acquire(1)
    213                  try:
    214                          self._conn.rollback()
    215                          self._conn.close()
    216                  except:
    217                          pass                            # connection can already be closed :-(
    218                  finally:
    219                          self._conn_lock.release()
    220          #-------------------------------
    221 -        def __start_thread(self):
    222                  if self._conn is None:
    223                          raise ValueError("no connection to backend available, useless to start thread")
    224
    225                  self._listener_thread = threading.Thread (
    226                          target = self._process_notifications,
    227                          name = self.__class__.__name__
    228                  )
    229                  self._listener_thread.setDaemon(True)
    230                  _log.info('starting listener thread')
    231                  self._listener_thread.start()
    232          #-------------------------------
    233          # the actual thread code
    234          #-------------------------------
    235 -        def _process_notifications(self):
    236
    237                  # get a cursor for this thread
    238                  self._conn_lock.acquire(1)
    239                  try:
    240                          self._cursor_in_thread = self._conn.cursor()
    241                  finally:
    242                          self._conn_lock.release()
    243
    244                  # loop until quitting
    245                  _have_quit_lock = None
    246                  while not _have_quit_lock:
    247
    248                          # quitting ?
    249                          if self._quit_lock.acquire(0):
    250                                  break
    251
    252                          # wait at most self._poll_interval for new data
    253                          self._conn_lock.acquire(1)
    254                          try:
    255                                  ready_input_sockets = select.select([self._conn_fd], [], [], self._poll_interval)[0]
    256                          finally:
    257                                  self._conn_lock.release()
    258
    259                          # any input available ?
    260                          if len(ready_input_sockets) == 0:
    261                                  # no, select.select() timed out
    262                                  # give others a chance to grab the conn lock (eg listen/unlisten)
    263                                  time.sleep(0.3)
    264                                  continue
    265
    266                          # data available, wait for it to fully arrive
    267  #                       while not self._cursor.isready():
    268  #                               pass
    269                          # replace by conn.poll() when psycopg2 2.2 becomes standard
    270                          self._conn_lock.acquire(1)
    271                          try:
    272                                  self._cursor_in_thread.execute(u'SELECT 1')
    273                                  self._cursor_in_thread.fetchall()
    274                          finally:
    275                                  self._conn_lock.release()
    276
    277                          # any notifications ?
    278                          while len(self._conn.notifies) > 0:
    279                                  # if self._quit_lock can be acquired we may be in
    280                                  # __del__ in which case gmDispatcher is not
    281                                  # guarantueed to exist anymore
    282                                  if self._quit_lock.acquire(0):
    283                                          _have_quit_lock = 1
    284                                          break
    285
    286                                  self._conn_lock.acquire(1)
    287                                  try:
    288                                          notification = self._conn.notifies.pop()
    289                                  finally:
    290                                          self._conn_lock.release()
    291                                  # try sending intra-client signal
    292                                  pid, full_signal = notification
    293                                  signal_name, pk = full_signal.split(':')
    294                                  try:
    295                                          results = gmDispatcher.send (
    296                                                  signal = signal_name,
    297                                                  originated_in_database = True,
    298                                                  listener_pid = self.backend_pid,
    299                                                  sending_backend_pid = pid,
    300                                                  pk_identity = pk
    301                                          )
    302                                  except:
    303                                          print "problem routing notification [%s] from backend [%s] to intra-client dispatcher" % (full_signal, pid)
    304                                          print sys.exc_info()
    305
    306                                  # there *may* be more pending notifications but do we care ?
    307                                  if self._quit_lock.acquire(0):
    308                                          _have_quit_lock = 1
    309                                          break
    310
    311                  # exit thread activity
    312                  return
    313  #=====================================================================
    314  # main
    315  #=====================================================================
    316  if __name__ == "__main__":
    317
    318          if len(sys.argv) < 2:
    319                  sys.exit()
    320
    321          if sys.argv[1] not in ['test', 'monitor']:
    322                  sys.exit()
    323
    324
    325          notifies = 0
    326
    327          from Gnumed.pycommon import gmPG2, gmI18N
    328          from Gnumed.business import gmPerson, gmPersonSearch
    329
    330          gmI18N.activate_locale()
    331          gmI18N.install_domain(domain='gnumed')
    332          #-------------------------------
    333 -        def run_test():
    334
    335                  #-------------------------------
    336                  def dummy(n):
    337                          return float(n)*n/float(1+n)
    338                  #-------------------------------
    339                  def OnPatientModified():
    340                          global notifies
    341                          notifies += 1
    342                          sys.stdout.flush()
    343                          print "\nBackend says: patient data has been modified (%s. notification)" % notifies
    344                  #-------------------------------
    345                  try:
    346                          n = int(sys.argv[2])
    347                  except:
    348                          print "You can set the number of iterations\nwith the second command line argument"
    349                          n = 100000
    350
    351                  # try loop without backend listener
    352                  print "Looping", n, "times through dummy function"
    353                  i = 0
    354                  t1 = time.time()
    355                  while i < n:
    356                          r = dummy(i)
    357                          i += 1
    358                  t2 = time.time()
    359                  t_nothreads = t2-t1
    360                  print "Without backend thread, it took", t_nothreads, "seconds"
    361
    362                  listener = gmBackendListener(conn = gmPG2.get_raw_connection())
    363
    364                  # now try with listener to measure impact
    365                  print "Now in a new shell connect psql to the"
    366                  print "database <gnumed_v9> on localhost, return"
    367                  print "here and hit <enter> to continue."
    368                  raw_input('hit <enter> when done starting psql')
    369                  print "You now have about 30 seconds to go"
    370                  print "to the psql shell and type"
    371                  print " notify patient_changed<enter>"
    372                  print "several times."
    373                  print "This should trigger our backend listening callback."
    374                  print "You can also try to stop the demo with Ctrl-C !"
    375
    376                  listener.register_callback('patient_changed', OnPatientModified)
    377
    378                  try:
    379                          counter = 0
    380                          while counter < 20:
    381                                  counter += 1
    382                                  time.sleep(1)
    383                                  sys.stdout.flush()
    384                                  print '.',
    385                          print "Looping",n,"times through dummy function"
    386                          i = 0
    387                          t1 = time.time()
    388                          while i < n:
    389                                  r = dummy(i)
    390                                  i += 1
    391                          t2 = time.time()
    392                          t_threaded = t2-t1
    393                          print "With backend thread, it took", t_threaded, "seconds"
    394                          print "Difference:", t_threaded-t_nothreads
    395                  except KeyboardInterrupt:
    396                          print "cancelled by user"
    397
    398                  listener.shutdown()
    399                  listener.unregister_callback('patient_changed', OnPatientModified)
    400          #-------------------------------
    401 -        def run_monitor():
    402
    403                  print "starting up backend notifications monitor"
    404
    405                  def monitoring_callback(*args, **kwargs):
    406                          try:
    407                                  kwargs['originated_in_database']
    408                                  print '==> got notification from database "%s":' % kwargs['signal']
    409                          except KeyError:
    410                                  print '==> received signal from client: "%s"' % kwargs['signal']
    411                          del kwargs['signal']
    412                          for key in kwargs.keys():
    413                                  print '    [%s]: %s' % (key, kwargs[key])
    414
    415                  gmDispatcher.connect(receiver = monitoring_callback)
    416
    417                  listener = gmBackendListener(conn = gmPG2.get_raw_connection())
    418                  print "listening for the following notifications:"
    419                  print "1) patient specific (patient #%s):" % listener.curr_patient_pk
    420                  for sig in listener.patient_specific_notifications:
    421                          print '   - %s' % sig
    422                  print "1) unspecific:"
    423                  for sig in listener.unspecific_notifications:
    424                          print '   - %s' % sig
    425
    426                  while True:
    427                          pat = gmPersonSearch.ask_for_patient()
    428                          if pat is None:
    429                                  break
    430                          print "found patient", pat
    431                          gmPerson.set_active_patient(patient=pat)
    432                          print "now waiting for notifications, hit <ENTER> to select another patient"
    433                          raw_input()
    434
    435                  print "cleanup"
    436                  listener.shutdown()
    437
    438                  print "shutting down backend notifications monitor"
    439
    440          #-------------------------------
    441          if sys.argv[1] == 'monitor':
    442                  run_monitor()
    443          else:
    444                  run_test()
    445
    446  #=====================================================================
    447