# # FHCRC InfoTech loghost swatch config file # # Don't use commas or apostrophe's in exec strings # # Global variable definitions perlcode my $echo = '/bin/echo'; perlcode my $mail_em = '/home/tocops/bin/mail_em'; perlcode my $mailx = '/bin/mailx'; perlcode my $page_em = '/home/tocops/bin/page_em'; perlcode my $pipe = '/home/tocops/.tocpipe'; #perlcode my $pipe = '/dev/null'; perlcode my $quota_em = '/opt/local/script/netapp_quotas/quota_notify.py'; ######################################################################## # Ignore long-running problems ######################################################################## ignore=/tuk-hot-emu/ ignore=/j2-ups/ ignore=/admaims22.*enclosure temperature has changed/ ######################################################################## # Ignore non-production gear ######################################################################## ignore=/charon-test/ ######################################################################## # Ignore high volume stuff ######################################################################## # Mail ignore=/imapd|pmx-milter|pop3d|postfix|HORDE/ # Hutch DC Authentication ignore=/dc-this security|dc-this netlogon|dc-this pnsclient/ # Hutch WINS ignore=/wluc\d\d wins/ # Catalyst link up/down ignore=/DTP-5-AUXVLANPORTO|SPANTREE-6-PORTFWD|LINK-3-UPDOWN|LINEPROTO-5-UPDOWN/ # IOS access-lists ignore=/SEC-6-IPACCESSLOGP/ # Wireless chatter ignore=/DOT11-6-DISASSOC|DOT11-6-DISASSOC|DOT11-7-AUTH_FAILED/ ignore=/DOT11-6-ROAMED|DOT11-4-MAXRETRIES/ # Management chatter ignore=/snmpd.*SNMP/ # VoIP chatter ignore=/\-dgw \d\d\d\d\d\d\d\d:/ # Named ignore=/named.*client.*denied|named.*update failed|named.*lame server resolving/ # INND ignore=/lists innd|lists nnrpd|lists ctlinnd/ # Perc Information Ignore ignore=/vxsvc_percpro\[info\]/ ignore=/The controller write policy has been changed/ # Windows stuff to ignore ignore=/evntagnt|termservdevices/ # Postgres ignore=/postgres/ # SMBd ignore=/nmbd|smbd/ # Automounter ignore=/automount|rpc.mountd/ # Miscellaneous Unixy apps ignore=/cron|pam_winbind|proftpd|snmpd|su:|sshd|syslog|xinetd|CRON/ # Miscellaneous Windowsy apps ignore=/groucho.*groupshield|isis.*groupshield/ ignore=/Recovery Storage Group/ # NetApp stuff to ignore ignore=/ 0 media errors| 0 checksum errors/ ignore=/Filer's security information differs from domain controller/ ignore=/NBT: Cannot connect NBNS socket to WINS server/ ignore=/Java_Thread/ ignore=/utf-8/ # BlueArc stuff to ignore ignore=/Severe:.*CIFS: EVS . cannot establish a connection to a DC/ ######################################################################## # Spawn pong-swatch if ping-swatch sends us a ping ######################################################################## watchfor=/Pinging swatch: are you there/ exec=/home/tocops/bin/pong-swatch ######################################################################## # Ignore ourselves. This line is critical; without it, the code which # swatch spawns to send mail and pages will trigger an endless # succession of mail and pages ######################################################################## ignore=/mail_em|page_em|--swatch|tocops/ # ######################################################################## # Tech Operations Console (TOC) ######################################################################## # Ignore stuff which I don't want to see on the TOC # ignore=/cf-vpn.*Protocol \[ SNMP \] attempted ADMIN logon/ ignore=/nagios:.*PING|nagios:.*Plugin timed out|nagios:.*NTP|nagios.*qpage/ ignore=/nagios.*YUM/ ignore=/nagios.*No data was recieved from host/ ignore=/nagios.*bigbird/ ignore=/apc Authentication Failure Trap/ ignore=/apcEnvSensorDisconnected/ ignore=/RTMT Alert Name:NumberOfRegisteredGateways/ # Populate apager window (bottom frame) watchfor=/apager.*(oncall-staff|vdops)/i exec=$echo apager $* > $pipe # Populate 911 tab watchfor=/CER_ONSITEALERT-4/ exec=$echo emergency $* > $pipe # Populate Ops (right hand frame) watchfor=/decartes nagios: SERVICE ALERT.*(CRITICAL;HARD|OK;HARD)|jamborite nagios: SERVICE EVENT HANDLER|iemHighTempThresholdViolation/ exec=$echo ops $* > $pipe continue # Populate NodeWatch (left hand frame) watchfor=/risier nodewatch:/ exec=$echo nodewatch $* > $pipe # Populate UPS log watchfor=/snmptrapd.*PowerNet-MIB/ exec=$echo ups $* > $pipe continue # Populate Hutch wireless log watchfor=/wirelesslog.*RealSource/ exec=$echo wireless $* > $pipe # Populate Guest wireless log watchfor=/daphnecaptiveportal.*cpl:|velmacaptiveportal.*cpl:/ exec=$echo wireless $* > $pipe # Populate Hutch IPSEC VPN log watchfor=/ASA.*AAA user authentication|ASA.*constructing blank hash payload|ASA.*AAA retrieved|ASA.*AAA transaction status|ASA.*Connection IPSec|ASA.*Security negotiation complete for|ASA.*ID_IPV4_ADDR ID|ASA.*Session disconnected|daphneVPN:|velmaVPN:/ exec=$echo ipsec $* > $pipe continue # Populate FHCRC webvpn log watchfor=/daphneWebVPN:|velmaWebVPN:|charon.*Connection terminated by the following DAP records:|charon.*SVC|charon.*WebVPN session/ exec=$echo webvpn $* > $pipe continue # Populate FHCRC webvpn detailed log watchfor=/daphneWebVPN:|velmaWebVPN:|charon.*DAP|charon.*AAA|charon.*SVC|charon.*WebVPN session/ exec=$echo webvpn_detailed $* > $pipe # Populate IPS log ignore=/edge-[a|b]-ips \[audit\]/ ignore=/edge-[a|b]-ips \d\d\d\d: [Ares|BitTorrent|Gnutella|IM|IRC|P2P|PPS]/ watchfor=/\d\d:\d\d:\d\d edge-[a|b]-ips/ exec=$echo ips $* > $pipe ######################################################################## # This is where I put 'ignore' strings when I am performing maintenace # work, and I want to prevent Swatch from emitting alarms. The # silence-swatch script uses this section, too --sk ######################################################################## # Begin Silence Section ignore=/j4-cns/ # End Silence Section ######################################################################## # Applications ######################################################################## # BIND issues watchfor=/CNAME and other data/ exec=$echo ops $* > $pipe mail=server-geeks-reports@fhcrc.org exec=$page_em ops-oncall CNAME/ANAME mismatch in zone file $7 --swatch throttle=60:00,use=regex watchfor=/unexpected rcode/ exec=$echo ops $* > $pipe mail=server-geeks-reports@fhcrc.org throttle=60.00,use=regex watchfor=/db_load could not open/ exec=$echo ops $* > $pipe mail=server-geeks-reports@fhcrc.org throttle=60:00,use=regex watchfor=/named.*permission denied/ exec=$echo ops $* > $pipe mail=server-geeks-reports@fhcrc.org throttle=60:00,use=regex ignore=/named/ # DHCP issues watchfor=/no free leases/ exec=$page_em oncall-staff A DHCP pool on $4 has exhausted its leases. --swatch exec=$echo ops $* > $pipe mail=server-geeks-reports@fhcrc.org,it-dts-notices throttle=480:00,use=regex #ignore=/dhcpd:/ # Sendmail issues ignore=/panthro.*connections.*daemon MTA/ watchfor=/rejecting connections on daemon MTA/ # exec=$echo ops $* > $pipe # exec=$page_em ops-oncall $4 is rejecting sendmail connections --swatch mail=server-geeks-reports@fhcrc.org throttle=15:00,use=regex watchfor=/accepting connections again for daemon MTA/ # exec=$echo ops $* > $pipe # exec=$page_em ops-oncall $4 is accepting sendmail connections --swatch mail=server-geeks-reports@fhcrc.org throttle=15:00,use=regex ignore=/ sendmail/ # argon Issues watchfor=/ALERT: \/argon_home is not mounted/ mail=server-geeks-reports@fhcrc.org exec=$page_em ops-oncall ALERT /argon_home is not mounted via NFS on $4 check NFS mount of /home --swatch watchfor=/RESTORED: \/argon_home is mounted/ mail=server-geeks-reports@fhcrc.org exec=$page_em ops-oncall RESTORED /argon_home is mounted via NFS on $4 --swatch watchfor=/Non-Zero mail files in \/var\/spool\/mail/ mail=server-geeks-reports@fhcrc.org exec=$page_em ops-oncall ALERT non-zero files in /var/spool/mail on $4 check NFS mount of /home --swatch watchfor=/\/var\/spool\/mail now contains only empty files/ mail=server-geeks-reports@fhcrc.org exec=$page_em ops-oncall RESTORED /var/spool/mail on $4 now contains only empty files --swatch ####################################################################### # Sniffer IP Address Discovery ####################################################################### # Example: May 19 12:48:14 dhcp1 dhcpd: DHCPACK on 10.25.75.157 to # 00:30:1b:af:99:c4 (apeman) via 10.25.74.3 # apeman watchfor=/to 00:30:1b:af:99:c4/ exec=$page_em bsmith apeman has an IP address of $8. --swatch threshold track_by="to 00:30:1b:af:99:c4", type=both, count=1, seconds=60 ######################################################################## # Firewall/IPS Stuff ######################################################################## ######################################################################## # Switch/Router/etc Hardware Errors ######################################################################## # Sup card failure (saw this once in dmz-b-rtr) watchfor=/disabling DCEF on linecard/ exec=$page_em oncall-staff The MSFC in $4 has suffered a hardware failure and is intermittently dropping packets. --swatch exec=$mail_em network-geeks The MSFC in $4 has suffered a hardware failure and is intermittently dropping packets. --swatch exec=$echo ops $* > $pipe throttle=480:00,use=regex # A module is failing watchfor=/timeout occurred|SYS-4-P2_WARN:1\/Astro|SYS-4-NVLOG:SYNDIAGS:Bus ASIC sync error/ exec=$page_em oncall-staff A module in $4 is failing or someone has created a loop using a mini-switch. --swatch exec=$mail_em network-geeks A module in $4 is failing or someone has created a loop using a mini-switch. --swatch exec=$echo ops $* > $pipe throttle=480:00,use=regex # A module is failing watchfor=/InbandPingProcessFailure|SYS-1-MOD_INVALIDSEQ:Bus asic invalid sequence occurred/ exec=$page_em oncall-staff A module in $4 is failing. --swatch exec=$mail_em network-geeks A module in $4 is failing. --swatch exec=$echo ops $* > $pipe throttle=480:00,use=regex # Ports on a module have failed # Example: Jul 16 18:06:16 j6-esx 2005 Jul 16 18:05:30 PDT -07:00 # %SYS-3-MOD_MINORFAIL:Minor problem in module 4 watchfor=/SYS-3-MOD_MINORFAIL/ exec=$page_em oncall-staff Module $15 in $4 has failed. --swatch exec=$mail_em network-geeks Module $15 in $4 has failed. --swatch exec=$echo ops $* > $pipe throttle=480:00,use=regex # Supervisor card is failing # Example: Nov 25 00:00:08 m4sr-a-esx 2005 Nov 25 00:00:07 Pacific -08:00 # %SYS-2-P2_CRIT: 1/Too many switching memory errors - about to # reset the system. watchfor=/Too many switching memory errors/ exec=$page_em oncall-staff The Supervisor card in $4 reports memory errors and will soon reset itself. --swatch exec=$mail_em network-geeks The Supervisor card in $4 reports memory errors and will soon reset itself. --swatch exec=$echo ops $* > $pipe throttle=120:00,use=regex # Supervisor card has failed # Example: Nov 25 00:00:09 m4sr-a-esx 2005 Nov 25 00:00:08 Pacific -08:00 # %SYS-2-P2_CRIT: 1/Please replace the Supervisor as soon as # possible watchfor=/Please replace the Supervisor as soon as possible/ exec=$page_em oncall-staff The Supervisor card in $4 has failed. --swatch exec=$mail_em network-geeks The Supervisor card in $4 has failed. --swatch exec=$echo ops $* > $pipe throttle=120:00,use=regex # Switch power supply failed watchfor=/Insufficient power supplies operating/ exec=$page_em oncall-staff A power supply in $4 has failed and $4 is no longer servicing its linecards. --swatch exec=$mail_em network-geeks A power supply in $4 has failed and $4 is no longer servicing its linecards. --swatch exec=$echo ops $* > $pipe throttle=120:00,use=regex # ASA HA Failure # Apr 27 10:33:59 charon-a-vpn-inside Apr 27 2009 10:33:59 charon-x-vpn : # %ASA-1-105005: (Secondary) Lost Failover communications with mate on # interface public-outside watchfor=/Lost Failover communications with mate/ exec=$echo ops $* > $pipe mail=network-geeks threshold track_by="Lost Failover communications with mate", type=both, count=1, seconds=3600 # Tipping Point hardware failure # Example: # System Log Notification (critical): Caught out of memory exception while # loading /usr/udm/ver3/base/signatures-meta.xml and then # System Log Notification (error): Low memory. UDM load request aborted. # System Log Notification (error): UDM load failed (120) watchfor=/System Log Notification/ exec=$echo ops $* > $pipe mail=network-geeks threshold track_by=System Log Notification, type=limit, count=1, seconds=3600 # Wireless Service Module (Wireless LAN Controller) hardware problems # From Cisco's dox: WiSM-5-RESET: Controller [dec]/[dec] not responding, # WiSM in slot [dec] is being reset watchfor=/WiSM-5-RESET: Controller.*not responding/ exec=$echo ops $* > $pipe mail=network-geeks threshold track_by=WiSM-5-RESET, type=both, count=1, seconds=3600 ######################################################################## # ISP Issues ######################################################################## # BGP flapping watchfor=/BGP-5-ADJCHANGE: neighbor \d+\.\d+\.\d+\.\d+ Down Interface flap/ mail=network-geeks threshold track_by="Down Interface flap", type=both, count=1, seconds=3600 ######################################################################## # Misconfiguration ######################################################################## # Routers log this message when they see another station claiming one # of their IP addresses. # example: # Jun 20 13:46:20 ga-a-rtr 534: 000431: Jun 20 13:46:19 pdt: %IP-4-DUPADDR: # Duplicate address 10.15.42.1 on Vlan242, sourced by 000e.0cb5.1c44 watchfor=/\%IP-4-DUPADDR: Duplicate address/ exec=$page_em ops-oncall $4 reports duplicate address $14 on $16. --swatch,when=1-5:8-17 exec=$mail_em network-geeks $4 reports duplicate address $14 on $16. --swatch exec=$echo ops $* > $pipe threshold track_by="$4 IP-4-DUPADDR", type=both, count=1, seconds=3600 # Duplex mismatch watchfor=/CDP-4-DUPLEX_MISMATCH/ exec=$echo ops $* > $pipe mail=network-geeks threshold track_by=CDP-4-DUPLEX_MISMATCH, type=both, count=10, seconds=3600 # Trunking misconfiguration watchfor=/DTP-1-ILGLCFG: Illegal config/ exec=$echo ops $* > $pipe mail=network-geeks threshold track_by=DTP-1-ILGLCFG: Illegal config, type=limit, count=1, seconds=3600 # Native VLAN Mismatch watchfor=/CDP-4-NATIVE_VLAN_MISMATCH: Native vlan mismatch/ exec=$echo ops $* > $pipe mail=network-geeks threshold track_by=CDP-4-NATIVE_VLAN_MISMATCH: Native vlan mismatch, type=both, count=10, seconds=28800 # Layer 2 Loop # Mar 22 05:17:30 dfsr-b-esx 52136: 052132: Mar 22 05:17:41 pdt: # %C4K_EBM-4-HOSTFLAPPING: Host 00:11:25:BF:71:68 in vlan 52 is flapping # between port Gi3/13 and port Gi3/14 ignore=/m3sr-a-esx.*C4K_EBM-4-HOSTFLAPPING/ watchfor=/C4K_EBM-4-HOSTFLAPPING/ exec=$echo ops $* > $pipe mail=network-geeks threshold track_by=C4K_EBM-4-HOSTFLAPPING, type=both, count=100, seconds=7200 watchfor=/unidirectional link detected/ exec=$page_em ops-oncall UDLD event detected on $4 on interface $15. --swatch,when=0-6:6-20 mail=network-geeks exec=$echo ops $* > $pipe threshold track_by="unidirectional link detected", type=both, count=1, seconds=3600 ######################################################################## # Resource Exhaustion ######################################################################## # Lack of memory watchfor=/SYS-2-MALLOCFAIL/ mail=network-geeks exec=$echo ops $* > $pipe throttle=60:00,use=regex # Hardware switching capacity exceeded # mmz-a-rtr 290: May 28 13:52:01 pdt: %MLSCEF-SP-4-FIB_EXCEPTION_THRESHOLD: # Hardware CEF entry usage is at 95% capacity for IPv4 unicast protocol. watchfor=/MLSCEF-4-FIB_EXCEPTION/ exec=$echo ops $* > $pipe mail=network-geeks threshold track_by=MLSCEF-4-FIB_EXCEPTION, type=limit, count=1, seconds=28800 # Software switching invoked # May 28 18:01:10 mmz-a-rtr 328: May 28 18:01:05 pdt: %CFIB-SP-7-CFIB_EXCEPTION: # FIB TCAM exception, Some entries will be software switched watchfor=/CFIB-SP-7-CFIB_EXCEPTION/ exec=$echo ops $* > $pipe mail=network-geeks threshold track_by=CFIB-SP-7-CFIB_EXCEPTION, type=limit, count=1, seconds=28800 # C4K TCAM Exhaustion watchfor=/C4K_HWACLMAN-4-ACLHWPROGERRREASON/ exec=$echo ops $* > $pipe mail=network-geeks threshold track_by=C4K_HWACLMAN-4-ACLHWPROGERRREASON, type=limit, count=1, seconds=28800 # Another TCAM Exhaustion message watchfor=/QM-4-TCAM_ENTRY/ exec=$echo ops $* > $pipe mail=network-geeks threshold track_by=QM-4-TCAM_ENTRY, type=limit, count=1, seconds=28800 # A NetFlow-related TCAM Exhaustion message watchfor=/EARL_NETFLOW-4-TCAM_THRLD/ exec=$echo ops $* > $pipe mail=network-geeks threshold track_by=EARL_NETFLOW-4-TCAM_THRLD, type=limit, count=1, seconds=28800 # CEF memory failure watchfor=/FIB-3-NOMEM/ exec=$echo ops $* > $pipe mail=network-geeks threshold track_by=FIB-3-NOMEM, type=limit, count=1, seconds=3600 # Hardware switching capacity exceeded watchfor=/MLSCEF-SP-7-FIB_EXCEPTION/ exec=$echo ops $* > $pipe mail=network-geeks threshold track_by=MLSCEF-SP-7-FIB_EXCEPTION, type=limit, count=1, seconds=28800 ######################################################################## # Access Layer Security ######################################################################## # Broadcast storm disable. # Mar 3 08:14:08 j4-esx 10113: 010109: Mar 3 08:14:16 pst: # %STORM_CONTROL-3-SHUTDOWN: A packet storm was detected on Gi5/29. The # interface has been disabled. watchfor=/STORM_CONTROL-3-SHUTDOWN/ exec=$echo ops $* > $pipe exec=$page_em ops-oncall $4 detected a broadcast storm on port $18 and has disabled this port. --swatch,when=1-5:6-18 exec=$mail_em network-geeks $4 detected a broadcast storm on port $18 and has disabled this port. --swatch threshold track_by=STORM_CONTROL-3-SHUTDOWN, type=limit, count=1, seconds=3600 # Broadcast storm suppression # Mar 3 08:27:28 j4-esx 10144: 010140: Mar 3 08:27:36 pst: # %STORM_CONTROL-3-FILTERED: A Broadcast storm detected on Gi5/29. A packet # filter action has been applied on the interface. watchfor=/STORM_CONTROL-3-FILTERED/ exec=$echo ops $* > $pipe exec=$mail_em network-geeks $4 detected a broadcast storm on port $18, throttling traffic. --swatch threshold track_by=STORM_CONTROL-3-FILTERED, type=limit, count=1, seconds=3600 # CAM table flood attack # Feb 16 10:31:56 j4-esx 80538: 080534: Feb 16 10:32:03 pst: # %PORT_SECURITY-2-PSECURE_VIOLATION: Security violation occurred, caused # by MAC address 0003.6bdd.2fa3 on port GigabitEthernet5/29. watchfor=/PORT_SECURITY-2-PSECURE_VIOLATION/ exec=$echo ops $* > $pipe exec=$mail_em network-geeks $4 detected a CAM flood attack on port $22. --swatch threshold track_by=PORT_SECURITY-2-PSECURE_VIOLATION, type=limit, count=1, seconds=3600 ######################################################################## # Acess Layer Hardware Failure ######################################################################## #Parity error detected in module #Apr 20 08:24:34 j4sr-a-esx 1306993: 1306993: Apr 20 08:24:34.457 pdt: #%PM_SCP-SP-1-LCP_FW_ERR: #System resetting module 2 to recover from error: #Linecard received system exception. Errcode = watchfor=/PM_SCP-SP-1-LCP_FW_ERR/ exec=$echo ops $* > $pipe exec=$page_em ops-oncall $4 detected a parity error in module $15. --swatch exec=mail_em network-geeks $4 detected a parity error in module $15. --swatch threshold trach_by=PM_SCP-SP-1-LCP_FW_ERR, type=limit, count=1, seconds=3600 ######################################################################## # Radius Servers ######################################################################## watchfor=/(DAPHNE|VELMA).*ERR: Could not create SSL/ exec=$echo ops $* > $pipe mail=network-geeks exec=$page_em oncall-staff Duty: $4 is having trouble authenticating wireless users. --swatch throttle=120:00,key="Could not create SSL" watchfor=/(DAPHNE|VELMA).*malloc failure/ exec=$echo ops $* > $pipe mail=network-geeks exec=$page_em oncall-staff Duty: $4 is having trouble authenticating wireless users. --swatch throttle=120:00,key="malloc failure" ######################################################################## # VoIP Monitoring ######################################################################## # VoIP / Voicemail watchfor=/Device unregistered. Device name.:CiscoUM1/ exec=$page_em ops-oncall A critical number of FHCRC Voicemail ports have unregistered. --swatch exec=$mail_em network-geeks A critical number of FHCRC Voicemail ports have unregistered. --swatch exec=$echo ops $* > $pipe threshold track_by="Device unregistered. Device name.:CiscoUM1", type=both, count=10, seconds=3600 watchfor=/CriticalServiceDown/ exec=$page_em ops-oncall A critical service is DOWN on $4. --swatch mail=network-geeks exec=$echo ops $* > $pipe threshold track_by=CriticalServiceDown, type=both, count=1, seconds=3600 watchfor=/HardwareFailure/ mail=network-geeks exec=$echo ops $* > $pipe threshold track_by=HardwareFailure, type=both, count=1, seconds=3600 watchfor=/SystemVersionMismatched/ mail=network-geeks exec=$echo ops $* > $pipe threshold track_by=SystemVersionMismatched, type=both, count=1, seconds=3600 watchfor=/DBReplicationFailure/ mail=network-geeks exec=$echo ops $* > $pipe threshold track_by=DBReplicationFailure, type=both, count=1, seconds=3600 watchfor=/CUC_SRM.*Fail/ exec=$page_em ops-oncall A failover event has occurred on $4. --swatch mail=network-geeks exec=$echo ops $* > $pipe threshold track_by=Fail, type=both, count=1, seconds=600 ignore=/CiscoLicenseApprochingLimit/ watchfor=/RTMT-ERROR-ALERT/ exec=$echo ops $* > $pipe threshold track_by=RTMT-ERROR-ALERT, type=both, count=1, seconds=3600 watchfor=/DeviceName:SEP002290BBF2B0/ exec=$page_em nsevilla Better check apeman. --swatch watchfor=/DeviceName:SEP0017E045B05E/ exec=$page_em nsevilla Better check apeman. --swatch # CER watchfor=/CER_DATABASE/ mail=network-geeks exec=$echo ops $* > $pipe threshold track_by=CER_DATABASE, type=both, count=1, seconds=3600 watchfor=/CER_AGGREGATOR/ exec=$echo ops $* > $pipe threshold track_by=CER_AGGREGATOR, type=both, count=1, seconds=3600 watchfor=/CER_CALLENGINE/ exec=$echo ops $* > $pipe threshold track_by=CER_CALLENGINE, type=both, count=1, seconds=3600 watchfor=/CER_GROUP/ exec=$echo ops $* > $pipe threshold track_by=CER_GROUP, type=both, count=1, seconds=3600 ######################################################################## # VPN Monitoring ######################################################################## watchfor=/This could indicate a pre-shared key mismatch/ exec=$page_em ops-oncall $4 is reporting a pre-shared key mismatch with IPSec VPN profile $14 . --swatch mail=network-geeks exec=$echo ops $* > $pipe threshold track_by="This could indicate a pre-shared key mismatch", type=both, count=1, seconds=3600 ######################################################################## # Site-to-site VPN Tunnels ######################################################################## # Watch for up/down events # Page for UM Tunnel Events watchfor=/mmz-a-rtr.*Crypto tunnel is DOWN.*Peer 128.34.95.1/ exec=$echo ops $* > $pipe exec=$page_em vdops The UM tunnel has gone down --swatch when=1-5:8-17 watchfor=/mmz-a-rtr.*Crypto tunnel is UP.*Peer 128.34.95.1/ exec=$echo ops $* > $pipe exec=$page_em vdops The UM tunnel has come back up --swatch when=1-5:8-17 # Page for pre-shared key mismatch watchfor=/CRYPTO-4-IKMP_BAD_MESSAGE/ mail=network-geeks Possible site-to-site VPN pre-shared key mismatch between $4 and $15. --swatch exec=$echo ops $* > $pipe threshold track_by=CRYPTO-4-IKMP_BAD_MESSAGE, type=both, count=10, seconds=3600 ######################################################################## # Firewalls - Nokia ######################################################################## watchfor=/systemTrapLowDiskSpace/ exec=$page_em ops-oncall $6 is running low on disk space. --swatch mail=network-geeks exec=$echo ops $* > $pipe throttle=60:00,use=regex watchfor=/systemTrapNoDiskSpace/ exec=$page_em ops-oncall $6 is out of disk space. --swatch mail=network-geeks exec=$echo ops $* > $pipe throttle=60:00,use=regex watchfor=/systemTrapDiskFailure/ exec=$page_em ops-oncall $6 has lost a disk. --swatch mail=network-geeks exec=$echo ops $* > $pipe throttle=60:00,use=regex watchfor=/systemTrapDiskMirrorSyncFailure/ exec=$page_em ops-oncall $6 is having disk problems. --swatch mail=network-geeks exec=$echo ops $* > $pipe throttle=60:00,use=regex watchfor=/systemOverTemperature/ exec=$page_em ops-oncall $6 is running hot. --swatch mail=network-geeks exec=$echo ops $* > $pipe throttle=60:00,use=regex watchfor=/systemPowerFailure/ exec=$page_em ops-oncall $6 has lost a power supply. --swatch mail=network-geeks exec=$echo ops $* > $pipe throttle=60:00,use=regex watchfor=/systemFanFailure/ exec=$page_em ops-oncall $6 has lost a fan. --swatch mail=network-geeks exec=$echo ops $* > $pipe throttle=60:00,use=regex watchfor=/systemTrapLicense/ exec=$page_em ops-oncall Licensed software on $6 has expired. --swatch mail=network-geeks exec=$echo ops $* > $pipe throttle=60:00,use=regex ######################################################################## # Power monitoring ######################################################################## # Hutchinson/Weintraub watchfor=/EMU:.*\'cf-pwr-mon\' closed/ exec=$echo ops $1 $2 $3 Hutchinson and Weintraub have lost utility power > $pipe exec=$page_em oncall-staff Hutchinson and Weintraub have lost utility power. --swatch,when=1-5:8-17 exec=$mail_em pwr-geeks Hutchinson and Weintraub have lost utility power. --swatch exec=$mail_em jlin Hutchinson and Weintraub have lost utility power. --swatch exec=$mail_em it-dts-notices Hutchinson and Weintraub have lost utility power. --swatch exec=$page_em sjones Hutchinson and Weintraub have lost utility power. --swatch exec=$page_em sjohnson Hutchinson and Weintraub have lost utility power. --swatch watchfor=/EMU:.*\'cf-pwr-mon\' opened/ exec=$echo ops $1 $2 $3 Hutchinson and Weintraub have regained utility power > $pipe exec=$page_em oncall-staff Hutchinson and Weintraub have regained utility power. --swatch,when=1-5:8-17 exec=$mail_em jlin Hutchinson and Weintraub have regained utility power. --swatch exec=$mail_em it-dts-notices Hutchinson and Weintraub have regained utility power. --swatch exec=$mail_em pwr-geeks Hutchinson and Weintraub have regained utility power. --swatch exec=$page_em sjones Hutchinson and Weintraub have regained utility power. --swatch exec=$page_em sjohnson Hutchinson and Weintraub have regained utility power. --swatch watchfor=/cf-emu System: Coldstart/ exec=$echo ops $1 $2 $3 cf-emu has coldstarted. --swatch # Thomas watchfor=/EMU:.*\'df-pwr-mon\' closed/ exec=$echo ops $1 $2 $3 Thomas has lost utility power > $pipe exec=$mail_em pwr-geeks Thomas has lost utility power. --swatch exec=$page_em sjones Thomas has lost utility power. --swatch exec=$page_em sjohnson Thomas has lost utility power. --swatch exec=$page_em oncall-staff Thomas has lost utility power. --swatch,when=1-5:8-17 watchfor=/EMU:.*\'df-pwr-mon\' opened/ exec=$echo ops $1 $2 $3 Thomas has regained utility power > $pipe exec=$mail_em pwr-geeks Thomas has regained utility power. --swatch exec=$page_em sjones Thomas has regained utility power. --swatch exec=$page_em sjohnson Thomas has regained utility power. --swatch exec=$page_em oncall-staff Thomas has regained utility power. --swatch,when=1-5:8-17 watchfor=/df-emu System: Coldstart/ exec=$echo ops $1 $2 $3 df-emu has coldstarted. --swatch # Yale watchfor=/EMU:.*\'ja-pwr-mon\' closed/ exec=$echo ops $1 $2 $3 Yale has lost utility power > $pipe exec=$page_em oncall-staff Yale has lost utility power. --swatch,when=1-5:8-17 exec=$mail_em ctrakarn Yale has lost utility power. --swatch exec=$page_em sjones Yale has lost utility power. --swatch exec=$page_em sjohnson Yale has lost utility power. --swatch exec=$mail_em pwr-geeks Yale has lost utility power. --swatch watchfor=/EMU:.*\'ja-pwr-mon\' opened/ exec=$echo ops $1 $2 $3 Yale has regained utility power > $pipe exec=$page_em oncall-staff Yale has regained utility power. --swatch,when=1-5:8-17 exec=$mail_em ctrakarn Yale has regained utility power. --swatch exec=$page_em sjones Yale has regained utility power. --swatch exec=$page_em sjohnson Yale has regained utility power. --swatch exec=$mail_em pwr-geeks Yale has regained utility power. --swatch watchfor=/ja-emu System: Coldstart/ exec=$echo ops $1 $2 $3 ja-emu has coldstarted. --swatch # 1616 Eastlake watchfor=/EMU:.*\'le4-pwr-mon\' closed/ exec=$echo ops $1 $2 $3 1616 Eastlake has lost utility power > $pipe exec=$mail_em pwr-geeks 1616 Eastlake has lost utility power. --swatch exec=$page_em sjones 1616 Eastlake has lost utility power. --swatch exec=$page_em sjohnson 1616 Eastlake has lost utility power. --swatch exec=$page_em oncall-staff 1616 Eastlake has lost utility power. --swatch,when=1-5:8-17 watchfor=/EMU:.*\'le4-pwr-mon\' opened/ exec=$echo ops $1 $2 $3 1616 Eastlake has regained utility power > $pipe exec=$mail_em pwr-geeks 1616 Eastlake has regained utility power. --swatch exec=$page_em sjones 1616 Eastlake has regained utility power. --swatch exec=$page_em sjohnson 1616 Eastlake has regained utility power. --swatch exec=$page_em oncall-staff 1616 Eastlake has regained utility power. --swatch,when=1-5:8-17 watchfor=/le4-emu System: Coldstart/ exec=$echo ops $1 $2 $3 le4-emu has coldstarted. --swatch # Arnold watchfor=/EMU:.*\'md-pwr-mon\' closed/ exec=$echo ops $1 $2 $3 Arnold has lost utility power > $pipe exec=$page_em oncall-staff Arnold has lost utility power. --swatch,when=1-5:8-17 exec=$mail_em pwr-geeks Arnold has lost utility power. --swatch exec=$mail_em ctrakarn Arnold has lost utility power. --swatch exec=$mail_em jlin Arnold has lost utility power. --swatch exec=$mail_em it-dts-notices Arnold has lost utility power. --swatch exec=$page_em sjones Arnold has lost utility power. --swatch exec=$page_em sjohnson Arnold has lost utility power. --swatch watchfor=/EMU:.*\'md-pwr-mon\' opened/ exec=$echo ops $1 $2 $3 Arnold has regained utility power > $pipe exec=$mail_em ctrakarn Arnold has regained utility power. --swatch exec=$mail_em jlin Arnold has regained utility power. --swatch exec=$mail_em it-dts-notices Arnold has regained utility power. --swatch exec=$mail_em pwr-geeks Arnold has regained utility power. --swatch exec=$page_em sjones Arnold has regained utility power. --swatch exec=$page_em sjohnson Arnold has regained utility power. --swatch exec=$page_em oncall-staff Arnold has regained utility power. --swatch,when=1-5:8-17 watchfor=/md-emu System: Coldstart/ exec=$echo ops $1 $2 $3 md-emu has coldstarted. --swatch # Hutch Kids watchfor=/hkids-ups.*upsOnBattery/ exec=$echo ops $1 $2 $3 Hutch Kids has lost utility power > $pipe exec=$mail_em pwr-geeks Hutch Kids has lost utility power. --swatch exec=$mail_em it-dts-notices Hutch Kids has lost utility power. --swatch exec=$page_em sjones Hutch Kids has lost utility power. --swatch exec=$page_em sjohnson Hutch Kids has lost utility power. --swatch exec=$page_em oncall-staff Hutch Kids has lost utility power. --swatch,when=1-5:8-17 watchfor=/hkids-ups.*powerRestored/ exec=$echo ops $1 $2 $3 Hutch Kids has regained utility power > $pipe exec=$mail_em pwr-geeks Hutch Kids has regained utility power. --swatch exec=$mail_em it-dts-notices Hutch Kids has regained utility power. --swatch exec=$page_em sjones Hutch Kids has regained utility power. --swatch exec=$page_em sjohnson Hutch Kids has regained utility power. --swatch exec=$page_em oncall-staff Hutch Kids has regained utility power. --swatch,when=1-5:8-17 # Fairview watchfor=/lf-ups.*upsOnBattery/ exec=$echo ops $1 $2 $3 Fairview has lost utility power > $pipe exec=$mail_em pwr-geeks Fairview has lost utility power. --swatch exec=$mail_em it-dts-notices Fairview has lost utility power. --swatch exec=$page_em sjones Fairview has lost utility power. --swatch exec=$page_em sjohnson Fairview has lost utility power. --swatch exec=$page_em oncall-staff Fairview has lost utility power. --swatch,when=1-5:8-17 watchfor=/lf-ups.*powerRestored/ exec=$echo ops $1 $2 $3 Fairview has regained utility power > $pipe exec=$mail_em pwr-geeks Fairview has regained utility power. --swatch exec=$mail_em it-dts-notices Fairview has regained utility power. --swatch exec=$page_em sjones Fairview has regained utility power. --swatch exec=$page_em sjohnson Fairview has regained utility power. --swatch exec=$page_em oncall-staff Fairview has regained utility power. --swatch,when=1-5:8-17 # Minor watchfor=/lm-ups.*upsOnBattery/ exec=$echo ops $1 $2 $3 Minor has lost utility power > $pipe exec=$mail_em pwr-geeks Minor has lost utility power. --swatch exec=$mail_em it-dts-notices Minor has lost utility power. --swatch exec=$page_em sjones Minor has lost utility power. --swatch exec=$page_em sjohnson Minor has lost utility power. --swatch exec=$page_em oncall-staff Minor has lost utility power. --swatch,when=1-5:8-17 watchfor=/lm-ups.*powerRestored/ exec=$echo ops $1 $2 $3 Minor has regained utility power > $pipe exec=$mail_em pwr-geeks Minor has regained utility power. --swatch exec=$mail_em it-dts-notices Minor has regained utility power. --swatch exec=$page_em sjones Minor has regained utility power. --swatch exec=$page_em sjohnson Minor has regained utility power. --swatch exec=$page_em oncall-staff Minor has regained utility power. --swatch,when=1-5:8-17 # Valley watchfor=/lv-ups.*upsOnBattery/ exec=$echo ops $1 $2 $3 Valley has lost utility power > $pipe exec=$mail_em pwr-geeks Valley has lost utility power. --swatch exec=$mail_em it-dts-notices Valley has lost utility power. --swatch exec=$page_em sjones Valley has lost utility power. --swatch exec=$page_em sjohnson Valley has lost utility power. --swatch exec=$page_em oncall-staff Valley has lost utility power. --swatch,when=1-5:8-17 watchfor=/lv-ups.*powerRestored/ exec=$echo ops $1 $2 $3 Valley has regained utility power > $pipe exec=$mail_em pwr-geeks Valley has regained utility power. --swatch exec=$mail_em it-dts-notices Valley has regained utility power. --swatch exec=$page_em sjones Valley has regained utility power. --swatch exec=$page_em sjohnson Valley has regained utility power. --swatch exec=$page_em oncall-staff Valley has regained utility power. --swatch,when=1-5:8-17 ######################################################################## # Environmental monitoring ######################################################################## watchfor=/Low humidity threshold violation for 'Temp Sensor/ #exec=$mail_em kwen $8 reports low humidity threshold violation at $63. --swatch watchfor=/High humidity threshold violation for 'Temp Sensor/ #exec=$mail_em kwen $4 reports high humidity threshold violation at $15 $16. --swatch watchfor=/High humidity threshold violation cleared for 'Temp Sensor/ exec=$mail_em kwen $4 reports high humidity threshold violation cleared at $16 $17. --swatch watchfor=/Low temperature threshold violation for 'Temp Sensor/ #exec=$mail_em kwen $8 reports $54 $55 $56 $57 $62 $63. --swatch watchfor=/Short-term increasing temperature rate of change violation for 'Temp Sensor/ #exec=$mail_em kwen &8 reports st inc temp rate of change. --swatch watchfor=/Short-term increasing temperature rate of change violation cleared for 'Temp Sensor/ #exec=$mail_em kwen &8 reports st inc temp rate of change cleared. --swatch ######################################################################## # Keybox monitoring ######################################################################## watchfor=/ja-emu EMU:.*\'j4-keysafe\' closed/ exec=$mailx -s "J4 Key Box Closed" mhuynh watchfor=/ja-emu EMU:.*'j4-keysafe' opened/ exec=$mailx -s "J4 Key Box Open" mhuynh ######################################################################## # UPS ######################################################################## # Skip little UPSes ignore=/10.25.120.7|le2-ups/ ignore=/10.25.122.7|le3s-ups/ ignore=/10.25.124.7|le4-1-ups/ ignore=/10.25.124.8|le4-2-ups/ ignore=/10.25.124.9|le4-3-ups/ ignore=/10.25.126.7|le5-ups/ ignore=/10.25.128.7|le5-test-ups/ ignore=/10.25.130.7|le3n-ups/ ignore=/10.25.16.7|a1-ups/ ignore=/10.25.18.7|a2-ups/ ignore=/10.25.196.7|annex-ups/ ignore=/10.25.198.7|rivkin-ups/ ignore=/10.25.20.7|a3-ups/ ignore=/10.25.200.4|hschool-ups/ ignore=/10.25.202.4|cabrini-ups/ ignore=/10.25.204.7|harrison-ups/ ignore=/10.25.22.7|b2-ups/ ignore=/10.25.24.7|b3-ups/ ignore=/10.25.26.7|c1-ups/ ignore=/10.25.28.7|c2-ups/ ignore=/10.25.30.7|c3-ups/ ignore=/10.25.32.7|bd-ups/ ignore=/10.25.36.7|lv-ups/ ignore=/10.25.38.7|lf-ups/ ignore=/10.25.46.7|lm-ups/ ignore=/10.25.48.7|hkids-ups/ ignore=/10.25.54.7|de-ups/ ignore=/10.25.56.7|d1-ups/ ignore=/10.25.58.7|d2-ups/ ignore=/10.25.60.7|d3-ups/ ignore=/10.25.142.123|dcisb1-1-ups/ ignore=/10.25.142.124|dcisb1-2-ups/ ignore=/10.25.142.133|dcisb2-1-ups/ ignore=/10.25.142.134|dcisb2-2-ups/ ignore=/10.25.142.143|dcisb3-1-ups/ ignore=/10.25.142.144|dcisb3-2-ups/ ignore=/10.25.142.153|dcisb4-1-ups/ ignore=/10.25.142.153|dcisb4-2-ups/ ignore=/10.25.62.7|d4-ups/ ignore=/10.25.64.7|d5-ups/ ignore=/10.25.68.7|j1-ups/ ignore=/10.25.68.8|j1-2-ups/ ignore=/10.25.70.7|j2-ups/ ignore=/10.25.72.7|j3-ups/ ignore=/10.25.74.7|j4-ups/ ignore=/10.25.76.7|j5-ups/ ignore=/10.25.78.7|j6-ups/ # Ignore humidity alarms from EMUs (I don't believe that they are accurate) ignore=/EMU:.*humidity/ # Only Mark and Todd are interested in the Silcons and Symmetras losing # utility power and recovering from a utility power loss watchfor=/PowerNet-MIB::(upsOnBattery|powerRestored)/ exec=$page_em sjones $* --swatch exec=$page_em sjohnson $* --swatch continue # Ignore Bypass Synchronization alarms ... these only occur during loss # of utility power, which we catch via another alarm ignore=/PowerNet-MIB::abnormalCondition.*Bypass Synchronization/ # Ignore BMS charging events -- these accompany loss of power, which we # capture using UPS-specific stanzas ignore=/PowerNet-MIB:bmsCriticalCondition.*String discharging in string/ ignore=/PowerNet-MIB:bmsCriticalConditionCleared.*String no longer discharging in string/ # Ignore unauthorized FTP attempts ignore=/PowerNet-MIB.*Detected an unauthorized user attempting to access the FTP interface/ # Ignore incomplete messages ignore=/PowerNet-MIB.*External 1/ # The auto-generated UPS section begins here # Look for UPS traps watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::upsInverterOverTemperature\)/ exec=$page_em pwr-geeks $* --swatch exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 upsInverterOverTemperature", type=limit, count=1, seconds=600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::dischargeCleared\)/ exec=$page_em pwr-geeks $* --swatch exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 dischargeCleared", type=limit, count=1, seconds=600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::switchedBypass\)/ exec=$page_em pwr-geeks $* --swatch exec=$mail_em pwr-geeks $* --swatch watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::batteryOverTemperatureCleared\)/ exec=$page_em pwr-geeks $* --swatch exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 batteryOverTemperatureCleared", type=limit, count=1, seconds=600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::upsPfcInputRelayFailure\)/ exec=$page_em pwr-geeks $* --swatch exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 upsPfcInputRelayFailure", type=limit, count=1, seconds=600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::upsOverloadCleared\)/ exec=$page_em pwr-geeks $* --swatch exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 upsOverloadCleared", type=limit, count=1, seconds=600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::returnFromBypass\)/ exec=$page_em pwr-geeks $* --swatch exec=$mail_em pwr-geeks $* --swatch watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::upsInverterFaultCleared\)/ exec=$page_em pwr-geeks $* --swatch exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 upsInverterFaultCleared", type=limit, count=1, seconds=600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::upsPfcInputRelayFailureCleared\)/ exec=$page_em pwr-geeks $* --swatch exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 upsPfcInputRelayFailureCleared", type=limit, count=1, seconds=600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::noBatteries\)/ exec=$page_em pwr-geeks $* --swatch exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 noBatteries", type=limit, count=1, seconds=600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::upsWokeUp\)/ exec=$page_em pwr-geeks $* --swatch exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 upsWokeUp", type=limit, count=1, seconds=600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::upsPfcFaultCleared\)/ exec=$page_em pwr-geeks $* --swatch exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 upsPfcFaultCleared", type=limit, count=1, seconds=600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::softwareBypass\)/ exec=$page_em pwr-geeks $* --swatch exec=$mail_em pwr-geeks $* --swatch watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::bmsCriticalCondition\)/ exec=$page_em pwr-geeks $* --swatch exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 bmsCriticalCondition", type=limit, count=1, seconds=600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::bmBatManBatteryAlarmCleared\)/ exec=$page_em pwr-geeks $* --swatch exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 bmBatManBatteryAlarmCleared", type=limit, count=1, seconds=600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::upsSleeping\)/ exec=$page_em pwr-geeks $* --swatch exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 upsSleeping", type=limit, count=1, seconds=600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::upsOverload\)/ exec=$page_em pwr-geeks $* --swatch exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 upsOverload", type=limit, count=1, seconds=600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::upsInverterFault\)/ exec=$page_em pwr-geeks $* --swatch exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 upsInverterFault", type=limit, count=1, seconds=600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::chargerFailure\)/ exec=$page_em pwr-geeks $* --swatch exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 chargerFailure", type=limit, count=1, seconds=600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::chargerFailureCleared\)/ exec=$page_em pwr-geeks $* --swatch exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 chargerFailureCleared", type=limit, count=1, seconds=600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::bypassPowerSupplyFailure\)/ exec=$page_em pwr-geeks $* --swatch exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 bypassPowerSupplyFailure", type=limit, count=1, seconds=600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::upsInternalOverTemperature\)/ exec=$page_em pwr-geeks $* --swatch exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 upsInternalOverTemperature", type=limit, count=1, seconds=600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::upsPfcFault\)/ exec=$page_em pwr-geeks $* --swatch exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 upsPfcFault", type=limit, count=1, seconds=600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::smwCriticalConditionCleared\)/ exec=$page_em pwr-geeks $* --swatch exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 smwCriticalConditionCleared", type=limit, count=1, seconds=600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::returnFromLowBattery\)/ exec=$page_em pwr-geeks $* --swatch exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 returnFromLowBattery", type=limit, count=1, seconds=600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::noBatteriesCleared\)/ exec=$page_em pwr-geeks $* --swatch exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 noBatteriesCleared", type=limit, count=1, seconds=600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::smwCriticalCondition\)/ exec=$page_em pwr-geeks $* --swatch exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 smwCriticalCondition", type=limit, count=1, seconds=600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::upsInternalOverTemperatureCleared\)/ exec=$page_em pwr-geeks $* --swatch exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 upsInternalOverTemperatureCleared", type=limit, count=1, seconds=600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::baseFanFailureCleared\)/ exec=$page_em pwr-geeks $* --swatch exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 baseFanFailureCleared", type=limit, count=1, seconds=600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::upsOutputSwitchOpened\)/ exec=$page_em pwr-geeks $* --swatch exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 upsOutputSwitchOpened", type=limit, count=1, seconds=600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::bmsCriticalConditionCleared\)/ exec=$page_em pwr-geeks $* --swatch exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 bmsCriticalConditionCleared", type=limit, count=1, seconds=600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::upsTurnedOff\)/ exec=$page_em pwr-geeks $* --swatch exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 upsTurnedOff", type=limit, count=1, seconds=600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::hardwareFailureBypass\)/ exec=$page_em pwr-geeks $* --swatch exec=$mail_em pwr-geeks $* --swatch watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::abnormalCondition\)/ exec=$page_em pwr-geeks $* --swatch exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 abnormalCondition", type=limit, count=1, seconds=600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::baseFanFailure\)/ exec=$page_em pwr-geeks $* --swatch exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 baseFanFailure", type=limit, count=1, seconds=600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::bmBatManBatteryAlarm\)/ exec=$page_em pwr-geeks $* --swatch exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 bmBatManBatteryAlarm", type=limit, count=1, seconds=600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::upsDischarged\)/ exec=$page_em pwr-geeks $* --swatch exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 upsDischarged", type=limit, count=1, seconds=600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::upsInverterOverTemperatureCleared\)/ exec=$page_em pwr-geeks $* --swatch exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 upsInverterOverTemperatureCleared", type=limit, count=1, seconds=600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::badVoltageCleared\)/ exec=$page_em pwr-geeks $* --swatch exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 badVoltageCleared", type=limit, count=1, seconds=600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::upsOutputSwitchClosed\)/ exec=$page_em pwr-geeks $* --swatch exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 upsOutputSwitchClosed", type=limit, count=1, seconds=600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::upsRebootStarted\)/ exec=$page_em pwr-geeks $* --swatch exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 upsRebootStarted", type=limit, count=1, seconds=600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::badVoltage\)/ exec=$page_em pwr-geeks $* --swatch exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 badVoltage", type=limit, count=1, seconds=600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::abnormalConditionCleared\)/ exec=$page_em pwr-geeks $* --swatch exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 abnormalConditionCleared", type=limit, count=1, seconds=600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::upsTurnedOn\)/ exec=$page_em pwr-geeks $* --swatch exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 upsTurnedOn", type=limit, count=1, seconds=600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::batteryOverTemperature\)/ exec=$page_em pwr-geeks $* --swatch exec=$mail_em pwr-geeks $* --swatch watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::upsBatteryTempSensorFaultCleared\)/ exec=$page_em pwr-geeks $* --swatch,when=1-5:7-18 exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 upsBatteryTempSensorFaultCleared", type=limit, count=1, seconds=3600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::upsBatterySoftBusStartFaultCleared\)/ exec=$page_em pwr-geeks $* --swatch,when=1-5:7-18 exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 upsBatterySoftBusStartFaultCleared", type=limit, count=1, seconds=3600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::powerModuleDecrease\)/ exec=$page_em pwr-geeks $* --swatch,when=1-5:7-18 exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 powerModuleDecrease", type=limit, count=1, seconds=3600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::rPDULowLoad\)/ exec=$page_em pwr-geeks $* --swatch,when=1-5:7-18 exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 rPDULowLoad", type=limit, count=1, seconds=3600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::intelligenceModuleInserted\)/ exec=$page_em pwr-geeks $* --swatch,when=1-5:7-18 exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 intelligenceModuleInserted", type=limit, count=1, seconds=3600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::upsBypassRelayFault\)/ exec=$page_em pwr-geeks $* --swatch,when=1-5:7-18 exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 upsBypassRelayFault", type=limit, count=1, seconds=3600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::temperatureThresholdViolation2\)/ exec=$page_em pwr-geeks $* --swatch,when=1-5:7-18 exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 temperatureThresholdViolation2", type=limit, count=1, seconds=3600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::batteryDecrease\)/ exec=$page_em pwr-geeks $* --swatch,when=1-5:7-18 exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 batteryDecrease", type=limit, count=1, seconds=3600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::upsBatterySoftBusStartFault\)/ exec=$page_em pwr-geeks $* --swatch,when=1-5:7-18 exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 upsBatterySoftBusStartFault", type=limit, count=1, seconds=3600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::intelligenceModuleRemoved\)/ exec=$page_em pwr-geeks $* --swatch,when=1-5:7-18 exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 intelligenceModuleRemoved", type=limit, count=1, seconds=3600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::upsBatteryTempSensorFault\)/ exec=$page_em pwr-geeks $* --swatch,when=1-5:7-18 exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 upsBatteryTempSensorFault", type=limit, count=1, seconds=3600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::powerModuleIncrease\)/ exec=$page_em pwr-geeks $* --swatch,when=1-5:7-18 exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 powerModuleIncrease", type=limit, count=1, seconds=3600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::rPDULowLoadCleared\)/ exec=$page_em pwr-geeks $* --swatch,when=1-5:7-18 exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 rPDULowLoadCleared", type=limit, count=1, seconds=3600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::extBatteryFrameDecrease\)/ exec=$page_em pwr-geeks $* --swatch,when=1-5:7-18 exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 extBatteryFrameDecrease", type=limit, count=1, seconds=3600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::smartRelayFaultCleared\)/ exec=$page_em pwr-geeks $* --swatch,when=1-5:7-18 exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 smartRelayFaultCleared", type=limit, count=1, seconds=3600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::upsBackfeedRelayFaultCleared\)/ exec=$page_em pwr-geeks $* --swatch,when=1-5:7-18 exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 upsBackfeedRelayFaultCleared", type=limit, count=1, seconds=3600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::temperatureThresholdViolationCleared2\)/ exec=$page_em pwr-geeks $* --swatch,when=1-5:7-18 exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 temperatureThresholdViolationCleared2", type=limit, count=1, seconds=3600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::upsBatteryOverVoltageCleared\)/ exec=$page_em pwr-geeks $* --swatch,when=1-5:7-18 exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 upsBatteryOverVoltageCleared", type=limit, count=1, seconds=3600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::externalSwitchGearOpened\)/ exec=$page_em pwr-geeks $* --swatch,when=1-5:7-18 exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 externalSwitchGearOpened", type=limit, count=1, seconds=3600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::externalSwitchGearClosed\)/ exec=$page_em pwr-geeks $* --swatch,when=1-5:7-18 exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 externalSwitchGearClosed", type=limit, count=1, seconds=3600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::upsBackfeedRelayFault\)/ exec=$page_em pwr-geeks $* --swatch,when=1-5:7-18 exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 upsBackfeedRelayFault", type=limit, count=1, seconds=3600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::temperatureThresholdViolationCleared1\)/ exec=$page_em pwr-geeks $* --swatch,when=1-5:7-18 exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 temperatureThresholdViolationCleared1", type=limit, count=1, seconds=3600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::extBatteryFrameIncease\)/ exec=$page_em pwr-geeks $* --swatch,when=1-5:7-18 exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 extBatteryFrameIncease", type=limit, count=1, seconds=3600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::passwordChanged\)/ exec=$page_em pwr-geeks $* --swatch,when=1-5:7-18 exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 passwordChanged", type=limit, count=1, seconds=3600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::temperatureThresholdViolation1\)/ exec=$page_em pwr-geeks $* --swatch,when=1-5:7-18 exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 temperatureThresholdViolation1", type=limit, count=1, seconds=3600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::upsBypassRelayFaultCleared\)/ exec=$page_em pwr-geeks $* --swatch,when=1-5:7-18 exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 upsBypassRelayFaultCleared", type=limit, count=1, seconds=3600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::upsBatteryOverVoltage\)/ exec=$page_em pwr-geeks $* --swatch,when=1-5:7-18 exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 upsBatteryOverVoltage", type=limit, count=1, seconds=3600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::batteryIncrease\)/ exec=$page_em pwr-geeks $* --swatch,when=1-5:7-18 exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 batteryIncrease", type=limit, count=1, seconds=3600 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::atsSourceSwitched\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 atsSourceSwitched", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::envMaxHumidityThresholdViolation\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 envMaxHumidityThresholdViolation", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::envSTIncTempRateViolationCleared\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 envSTIncTempRateViolationCleared", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::rPDUPowerSupply2Fail\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 rPDUPowerSupply2Fail", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::envSTDecTempRateViolation\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 envSTDecTempRateViolation", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::humidityThresholdViolationCleared1\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 humidityThresholdViolationCleared1", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::bmBatManMaintenanceAlarm\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 bmBatManMaintenanceAlarm", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::rPDUPowerSupply1Fail\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 rPDUPowerSupply1Fail", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::bmBatManEnvironmentAlarm\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 bmBatManEnvironmentAlarm", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::envLTDecTempRateViolation\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 envLTDecTempRateViolation", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::bmBatManMaintenanceAlarmCleared\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 bmBatManMaintenanceAlarmCleared", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::humidityThresholdViolation1\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 humidityThresholdViolation1", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::bmBatManCommEstab\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 bmBatManCommEstab", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::envLTIncTempRateViolation\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 envLTIncTempRateViolation", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::bmsWarningConditionCleared\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 bmsWarningConditionCleared", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::envLTIncTempRateViolationCleared\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 envLTIncTempRateViolationCleared", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::atsPowerSupplyFailureCleared\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 atsPowerSupplyFailureCleared", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::atsRedundancyRestored\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 atsRedundancyRestored", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::batteryPackCommLost\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 batteryPackCommLost", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::envLowTempThresholdViolation\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 envLowTempThresholdViolation", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::calibrationStart\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 calibrationStart", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::envMinHumidityThresholdViolationCleared\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 envMinHumidityThresholdViolationCleared", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::atsPowerSupplyFailure\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 atsPowerSupplyFailure", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::apcEnvSensorConnected\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 apcEnvSensorConnected", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::smartBoostOff\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 smartBoostOff", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::iemContactFault\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 iemContactFault", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::atsOverCurrentCleared\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 atsOverCurrentCleared", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::batteryPackCommEstablished\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 batteryPackCommEstablished", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::humidityThresholdViolationCleared2\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 humidityThresholdViolationCleared2", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::iemLowHumidThresholdViolation\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 iemLowHumidThresholdViolation", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::humidityThresholdViolation2\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 humidityThresholdViolation2", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::iemLowTempThresholdViolation\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 iemLowTempThresholdViolation", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::envMinHumidityThresholdViolation\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 envMinHumidityThresholdViolation", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::upsBatteryNeedsReplacement\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 upsBatteryNeedsReplacement", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::envHighTempThresholdViolation\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 envHighTempThresholdViolation", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::upsCalibrationStackChanged\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 upsCalibrationStackChanged", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::atsCommunicationEstablished\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 atsCommunicationEstablished", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::rPDUPowerSupply10k\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 rPDUPowerSupply10k", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::envLTDecTempRateViolationCleared\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 envLTDecTempRateViolationCleared", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::gracefullShutdown\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 gracefullShutdown", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::rPDUPowerSupply20k\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 rPDUPowerSupply20k", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::envLowTempThresholdViolationCleared\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 envLowTempThresholdViolationCleared", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::powerRestored\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 powerRestored", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::iemHighTempThresholdViolationCleared\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 iemHighTempThresholdViolationCleared", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::atsOverCurrent\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 atsOverCurrent", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::atsLostRedundancy\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 atsLostRedundancy", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::bmBatManCommLost\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 bmBatManCommLost", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::envMaxTempThresholdViolation\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 envMaxTempThresholdViolation", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::envMaxHumidityThresholdViolationCleared\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 envMaxHumidityThresholdViolationCleared", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::iemHighHumidThresholdViolation\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 iemHighHumidThresholdViolation", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::envMinTempThresholdViolationCleared\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 envMinTempThresholdViolationCleared", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::upsOnBattery\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 upsOnBattery", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::iemLowHumidThresholdViolationCleared\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 iemLowHumidThresholdViolationCleared", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::iemLowTempThresholdViolationCleared\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 iemLowTempThresholdViolationCleared", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::bmsWarningCondition\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 bmsWarningCondition", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::iemContactFaultCleared\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 iemContactFaultCleared", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::upsDiagnosticsFailed\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 upsDiagnosticsFailed", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::smartAvrReducing\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 smartAvrReducing", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::apcEnvSensorDisconnected\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 apcEnvSensorDisconnected", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::upsBatteryReplaced\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 upsBatteryReplaced", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::smartAvrReducingOff\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 smartAvrReducingOff", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::envLowHumidityThresholdViolation\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 envLowHumidityThresholdViolation", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::envSTDecTempRateViolationCleared\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 envSTDecTempRateViolationCleared", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::envHighHumidityThresholdViolationCleared\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 envHighHumidityThresholdViolationCleared", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::bmBatManKneeAlarmCleared\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 bmBatManKneeAlarmCleared", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::envHighTempThresholdViolationCleared\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 envHighTempThresholdViolationCleared", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::calibrationEnd\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 calibrationEnd", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::bmBatManKneeAlarm\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 bmBatManKneeAlarm", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::envSTIncTempRateViolation\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 envSTIncTempRateViolation", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::envMaxTempThresholdViolationCleared\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 envMaxTempThresholdViolationCleared", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::envMinTempThresholdViolation\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 envMinTempThresholdViolation", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::atsCommunicationLost\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 atsCommunicationLost", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::iemHighTempThresholdViolation\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 iemHighTempThresholdViolation", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::upsDipSwitchChanged\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 upsDipSwitchChanged", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::envLowHumidityThresholdViolationCleared\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 envLowHumidityThresholdViolationCleared", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::bmBatManEnvironmentAlarmCleared\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 bmBatManEnvironmentAlarmCleared", type=limit, count=1, seconds=28800 watchfor=/$8.*Enterprise Specific Trap \(PowerNet-MIB::iemHighHumidThresholdViolationCleared\)/ exec=$mail_em pwr-geeks $* --swatch threshold track_by="$8 iemHighHumidThresholdViolationCleared", type=limit, count=1, seconds=28800 # The auto-generated UPS section terminates here ######################################################################## # Unix OS Stuff ######################################################################## # File system full watchfor=/file system full/ exec=$page_em ops-oncall Disk space exhausted on $4. --swatch exec=$echo ops $* > $pipe mail=server-geeks-reports@fhcrc.org throttle=480:00,use=regex # System crashes watchfor=/unix:.*panic/ exec=$page_em ops-oncall $4 panicked and is now rebooting. --swatch mail=server-geeks-reports@fhcrc.org throttle=480:00,use=regex # File system errors watchfor=/Media Error/ exec=$page_em ops-oncall Disk problems on $4. --swatch exec=$echo ops $* > $pipe mail=server-geeks-reports@fhcrc.org throttle=480:00,key="Media Error" # Someone turned AppArmor on watchfor=/SubDomain: REJECTING/ exec=$page_em server-geeks AppArmor might be running on $4. Please check and disable. --swatch exec=$echo ops $* > $pipe mail=server-geeks-reports@fhcrc.org throttle=480:00,key="SubDomain: REJECTING" ######################################################################## # Unix Hardware errors ######################################################################## # Memory errors watchfor=/dma error|DMA error/ exec=$page_em ops-oncall RAM problems on $4. --swatch exec=$echo ops $* > $pipe mail=server-geeks-reports@fhcrc.org throttle=480:00,use=regex # SCSI Bus errors watchfor=/SCSI transport failed/ exec=$page_em ops-oncall SCSI bus problems on $4. --swatch exec=$echo ops $* > $pipe mail=server-geeks-reports@fhcrc.org throttle=480:00,key="SCSI transport failed" # aacraid errors watchfor=/aacraid.*Error/ exec=$page_em ops-oncall Drive error: $6 on $4. --swatch exec=$echo ops $* > $pipe mail=server-geeks-reports@fhcrc.org throttle=480:00,key=regex watchfor=/aacraid.*failure/ exec=$page_em ops-oncall Drive failure: $6 on $4. --swatch exec=$echo ops $* > $pipe mail=server-geeks-reports@fhcrc.org throttle=480:00,key=regex watchfor=/AAC:.*Error/ exec=$page_em ops-oncall Drive error: $6 on $4. --swatch exec=$echo ops $* > $pipe mail=server-geeks-reports@fhcrc.org throttle=480:00,key=regex watchfor=/AAC:.*failed/ exec=$page_em ops-oncall Drive failure: $6 on $4. --swatch exec=$echo ops $* > $pipe mail=server-geeks-reports@fhcrc.org throttle=480:00,key=regex watchfor=/AAC:.*Failure/ exec=$page_em ops-oncall Drive failure: $6 on $4. --swatch exec=$echo ops $* > $pipe mail=server-geeks-reports@fhcrc.org throttle=480:00,key=regex watchfor=/Read-only file system/ exec=$page_em ops-oncall Read-only file system on $4. --swatch exec=$echo ops $* > $pipe mail=server-geeks-reports@fhcrc.org threshold track_by=$4,type=limit,count=1,seconds=1800 ######################################################################## # Titan Alerts ######################################################################## watchfor=/aristotle (\d\d\d\d) Severe.*OPEN_TAPE_DEVICE/ mail=server-geeks-reports@fhcrc.org exec=$echo ops $* > $pipe exec=$page_em ops-oncall $* --swatch, when=8-16 throttle=480:00,key="FSA2 has been waiting for call OPEN_TAPE_DEVICE" watchfor=/aristotle (\d\d\d\d) Severe:/ mail=server-geeks-reports@fhcrc.org exec=$echo ops $* > $pipe exec=$page_em ops-oncall $* --swatch throttle=480:00,key="aristotle $5 Severe" watchfor=/aristotle (\d\d\d\d) Critical:/ mail=server-geeks-reports@fhcrc.org exec=$echo ops $* > $pipe exec=$page_em ops-oncall $* --swatch throttle=480:00,key="aristotle $5 Critical" watchfor=/aristotle (\d\d\d\d) Warning: No big free heap blocks left:/ mail=server-geeks-reports@fhcrc.org exec=$echo ops $* > $pipe exec=$page_em ops-oncall $* --swatch throttle=480:00,key="aristotle $5 Critical" watchfor=/aristotle (\d\d\d\d) Warning:.*disk.*failed/ mail=server-geeks-reports@fhcrc.org exec=$echo ops $* > $pipe exec=$page_em ops-oncall $* --swatch throttle=480:00,key="aristotle $5 Critical" watchfor=/philo (\d\d\d\d) Severe:/ mail=server-geeks-reports@fhcrc.org exec=$echo ops $* > $pipe exec=$page_em ops-oncall $* --swatch throttle=480:00,key="philo $5 Severe" watchfor=/philo (\d\d\d\d) Critical:/ mail=server-geeks-reports@fhcrc.org exec=$echo ops $* > $pipe exec=$page_em ops-oncall $* --swatch throttle=480:00,key="philo $5 Critical" watchfor=/philo (\d\d\d\d) Warning: No big free heap blocks left:/ mail=server-geeks-reports@fhcrc.org exec=$echo ops $* > $pipe exec=$page_em ops-oncall $* --swatch throttle=480:00,key="philo $5 Critical" watchfor=/philo (\d\d\d\d) Warning:.*disk.*failed/ mail=server-geeks-reports@fhcrc.org exec=$echo ops $* > $pipe exec=$page_em ops-oncall $* --swatch throttle=480:00,key="philo $5 Critical" watchfor=/arestes (\d\d\d\d) Severe:/ mail=server-geeks-reports@fhcrc.org exec=$echo ops $* > $pipe exec=$page_em ops-oncall $* --swatch throttle=480:00,key="arestes $5 Severe" watchfor=/arestes (\d\d\d\d) Critical:/ mail=server-geeks-reports@fhcrc.org exec=$echo ops $* > $pipe exec=$page_em ops-oncall $* --swatch throttle=480:00,key="arestes $5 Critical" watchfor=/arestes (\d\d\d\d) Warning: No big free heap blocks left:/ mail=server-geeks-reports@fhcrc.org exec=$echo ops $* > $pipe exec=$page_em ops-oncall $* --swatch throttle=480:00,key="arestes $5 Critical" watchfor=/arestes (\d\d\d\d) Warning:.*disk.*failed/ mail=server-geeks-reports@fhcrc.org exec=$echo ops $* > $pipe exec=$page_em ops-oncall $* --swatch throttle=480:00,key="arestes $5 Critical" watchfor=/arestespublic (\d\d\d\d) Severe:/ mail=server-geeks-reports@fhcrc.org exec=$echo ops $* > $pipe exec=$page_em ops-oncall $* --swatch throttle=480:00,key="arestes $5 Severe" watchfor=/arestespublic (\d\d\d\d) Critical:/ mail=server-geeks-reports@fhcrc.org exec=$echo ops $* > $pipe exec=$page_em ops-oncall $* --swatch throttle=480:00,key="arestes $5 Critical" watchfor=/arestespublic (\d\d\d\d) Warning: No big free heap blocks left:/ mail=server-geeks-reports@fhcrc.org exec=$echo ops $* > $pipe exec=$page_em ops-oncall $* --swatch throttle=480:00,key="arestes $5 Critical" watchfor=/arestespublic (\d\d\d\d) Warning:.*disk.*failed/ mail=server-geeks-reports@fhcrc.org exec=$echo ops $* > $pipe exec=$page_em ops-oncall $* --swatch throttle=480:00,key="arestes $5 Critical" watchfor=/pliny (\d\d\d\d) Severe:/ mail=server-geeks-reports@fhcrc.org exec=$echo ops $* > $pipe exec=$page_em ops-oncall $* --swatch throttle=480:00,key="pliny $5 Severe" watchfor=/pliny (\d\d\d\d) Critical:/ mail=server-geeks-reports@fhcrc.org exec=$echo ops $* > $pipe exec=$page_em ops-oncall $* --swatch throttle=480:00,key="pliny $5 Critical" watchfor=/pliny (\d\d\d\d) Warning: No big free heap blocks left:/ mail=server-geeks-reports@fhcrc.org exec=$echo ops $* > $pipe exec=$page_em ops-oncall $* --swatch throttle=480:00,key="pliny $5 Critical" watchfor=/pliny (\d\d\d\d) Warning:.*disk.*failed/ mail=server-geeks-reports@fhcrc.org exec=$echo ops $* > $pipe exec=$page_em ops-oncall $* --swatch throttle=480:00,key="pliny $5 Critical" ######################################################################## # SCSI errors ######################################################################## # SCSI RAID errors watchfor=/SCSI error/ exec=$page_em ops-oncall $4 reports a scsi problem. Msg is: $* --swatch mail=server-geeks-reports@fhcrc.org exec=$echo ops $* > $pipe throttle=480:00,use=regex watchfor=/scsi.*error/ exec=$page_em ops-oncall $4 reports a scsi problem. Msg is: $* --swatch mail=server-geeks-reports@fhcrc.org exec=$echo ops $* > $pipe throttle=480:00,use=regex watchfor=/PERC.*failed/ exec=$page_em ops-oncall $4 reports a drive failure. Msg is: $* --swatch mail=server-geeks-reports@fhcrc.org exec=$echo ops $* > $pipe throttle=480:00,use=regex watchfor=/PERC.*degraded/ exec=$page_em ops-oncall $4 reports a drive problem. Msg is: $* --swatch mail=server-geeks-reports@fhcrc.org exec=$echo ops $* > $pipe throttle=480:00,use=regex watchfor=/PERC.*removed/ exec=$page_em ops-oncall $4 reports a drive problem. Msg is: $* --swatch mail=server-geeks-reports@fhcrc.org exec=$echo ops $* > $pipe throttle=480:00,use=regex watchfor=/vxsvc_percpro/ exec=$page_em ops-oncall $4 error: $* --swatch mail=server-geeks-reports@fhcrc.org exec=$echo ops $* > $pipe throttle=480:00,use=regex watchfor=/vxsvc_alert/ exec=$page_em ops-oncall $4 error: $* --swatch mail=server-geeks-reports@fhcrc.org exec=$echo ops $* > $pipe throttle=480:00,use=regex watchfor=/administrator\[info/ mail=server-geeks-reports@fhcrc.org exec=$echo ops $* > $pipe throttle=480:00,use=regex watchfor=/administrator\[warn/ exec=$page_em ops-oncall $4 error: $* --swatch mail=server-geeks-reports@fhcrc.org exec=$echo ops $* > $pipe throttle=480:00,use=regex watchfor=/administrator\[error/ exec=$page_em ops-oncall $4 error: $* --swatch mail=server-geeks-reports@fhcrc.org exec=$echo ops $* > $pipe throttle=480:00,use=regex watchfor=/Server Administrator.*Device failed/ exec=$page_em ops-oncall $4 error: $* --swatch mail=server-geeks-reports@fhcrc.org exec=$echo ops $* > $pipe throttle=480:00,use=regex # 2094 = Predictive Failure -- Replace drive watchfor=/Server Administrator: Storage Service EventID: 2094/ exec=$page_em ops-oncall $4 error: $* --swatch mail=server-geeks-reports@fhcrc.org exec=$echo ops $* > $pipe throttle=480:00,use=regex # 2095 = SCSI Sense Data -- Replace drive watchfor=/Server Administrator: Storage Service EventID: 2095/ exec=$page_em ops-oncall $4 error: $* --swatch mail=server-geeks-reports@fhcrc.org exec=$echo ops $* > $pipe throttle=480:00,use=regex ######################################################################## # Exchange errors ######################################################################## watchfor=/ 9539.*stopped/ exec=$page_em ops-oncall $4 error: $* --swatch mail=server-geeks-reports@fhcrc.org exec=$echo ops $* > $pipe watchfor=/ 9523.*started/ exec=$page_em ops-oncall $4 recovery: $* --swatch mail=server-geeks-reports@fhcrc.org exec=$echo ops $* > $pipe watchfor=/msexchangesa\[error\]\ 9175/ exec=$page_em ops-oncall $4 error: $* --swatch mail=server-geeks-reports@fhcrc.org exec=$echo ops $* > $pipe ######################################################################## # WebMail Spam Detection ######################################################################## watchfor=/webmail in.imapproxyd/ exec=$page_em jkuth Possible spam attack on $4 --swatch mail=jkuth exec=$echo ops $* > $pipe threshold type=both, count=400 , seconds=600 ######################################################################## # Data Domain errors ######################################################################## watchfor=/Disk [0-9]+ is failed and should be replaced/ exec=$page_em ops-oncall $4 error: $* --swatch mail=server-geeks-reports@fhcrc.org exec=$echo ops $* > $pipe throttle=480:00,use=regex watchfor=/Disk [0-9]+ is absent and should be replaced/ exec=$page_em ops-oncall $4 error: $* --swatch mail=server-geeks-reports@fhcrc.org exec=$echo ops $* > $pipe throttle=480:00,use=regex watchfor=/A power supply module has failed/ exec=$page_em ops-oncall $4 error: $* --swatch mail=server-geeks-reports@fhcrc.org exec=$echo ops $* > $pipe throttle=480:00,use=regex ######################################################################## # NetApp ######################################################################## watchfor=/ltm_services.*Link down/ exec=$page_em ops-oncall $4 error: $* --swatch mail=server-geeks-reports@fhcrc.org exec=$echo ops $* > $pipe throttle=480:00,use=regex watchfor=/tungsten.*quota.softlimit.exceeded/ exec=$quota_em $* ######################################################################## # 3Par ######################################################################## watchfor=/handel comp_state_change.*Degraded/ exec=$page_em ops-oncall $4 error: $* --swatch mail=server-geeks-reports@fhcrc.org exec=$echo ops $* > $pipe threshold track_by=Degraded,type=limit,count=1,seconds=600 watchfor=/handel comp_state_change.*Failed/ exec=$page_em ops-oncall $4 error: $* --swatch mail=server-geeks-reports@fhcrc.org exec=$echo ops $* > $pipe threshold track_by=Failed,type=limit,count=1,seconds=600