use bpm to run blackbox

jobs/syslog_forwarder/monit

@@ -1,9 +1,9 @@
 <% unless p('syslog.migration.disabled') %>
 <% if p('syslog.forward_files') %>
 check process blackbox
-  with pidfile /var/vcap/sys/run/syslog_forwarder/blackbox/blackbox.pid
-  start program "/var/vcap/jobs/syslog_forwarder/bin/blackbox_ctl start"
-  stop program "/var/vcap/jobs/syslog_forwarder/bin/blackbox_ctl stop"
+  with pidfile /var/vcap/sys/run/bpm/syslog_forwarder/blackbox.pid
+  start program "/var/vcap/jobs/bpm/bin/bpm start syslog_forwarder -p blackbox"
+  stop program "/var/vcap/jobs/bpm/bin/bpm stop syslog_forwarder -p blackbox"
   group vcap
 <% end %>
 <% end %>

jobs/syslog_forwarder/spec

@@ -2,7 +2,7 @@
 name: syslog_forwarder
 
 templates:
-  blackbox_ctl.erb: bin/blackbox_ctl
+  bpm.yml.erb: config/bpm.yml
   blackbox_config.yml.erb: config/blackbox_config.yml
   ca_cert.pem.erb: config/ca_cert.pem
   client.crt.erb: config/client.crt

jobs/syslog_forwarder/templates/bpm.yml.erb

@@ -0,0 +1,20 @@
+<% unless p('syslog.migration.disabled') || !p('syslog.forward_files') -%>
+processes:
+- name: blackbox
+  executable: /var/vcap/packages/blackbox/bin/blackbox
+  args:
+  - -config=/var/vcap/jobs/syslog_forwarder/config/blackbox_config.yml
+<% unless p('syslog.respect_file_permissions') -%>
+  capabilities:
+  - DAC_READ_SEARCH
+<% end -%>
+<% if p('syslog.blackbox.limit_cpu') -%>
+  env:
+    GOMAXPROCS: "1"
+<% end -%>
+  unsafe:
+    unrestricted_volumes:
+    - path: <%= p("syslog.blackbox.source_dir") %>
+      writable: false
+      mount_only: true
+<% end -%>

jobs/syslog_storer/templates/syslog.apparmor.erb

@@ -1,3 +1,6 @@
 # syslog_storer rules
+    /var/vcap/data/jobs/syslog_storer/*/config/* r,
+    /var/vcap/jobs/syslog_storer/config/* r,
+    /var/vcap/data/syslog_storer/** rw,
     /var/vcap/store/syslog_storer/ rw,
     /var/vcap/store/syslog_storer/** rw,

scripts/test

@@ -9,6 +9,7 @@ pushd "$(dirname "$0")/.."
   bosh upload-release
 popd
 
+bosh upload-release https://bosh.io/d/github.com/cloudfoundry/bpm-release
 bosh upload-stemcell "https://bosh.io/d/stemcells/bosh-google-kvm-${STEMCELL_OS}-go_agent"
 
 pushd "$(dirname "$0")/../tests"

tests/acceptance_test.go

@@ -127,6 +127,17 @@ var _ = Describe("Forwarding loglines to a TCP syslog drain", func() {
 			})
 		})
 
+		Context("when a file is created in a different job's log directory", func() {
+			BeforeEach(func() {
+				session := ForwarderSshCmd("sudo mkdir -p /var/vcap/sys/log/other-job && sudo chmod 777 /var/vcap/sys/log/other-job && sudo touch /var/vcap/sys/log/other-job/app.log")
+				Eventually(session).Should(gexec.Exit(0))
+			})
+
+			It("forwards new lines written to the file through syslog", func() {
+				Eventually(WriteToOtherJobTestFile("test-other-job-forwarding")).Should(gbytes.Say("test-other-job-forwarding"))
+			})
+		})
+
 		It("has a valid config", func() {
 			session := ForwarderSshCmd("sudo rsyslogd -N1")
 			Eventually(session).Should(gexec.Exit(0))

tests/boshhelpers_test.go

@@ -128,6 +128,14 @@ func WriteToTestFile(message string) func() *gexec.Session {
 	}
 }
 
+func WriteToOtherJobTestFile(message string) func() *gexec.Session {
+	return func() *gexec.Session {
+		session := ForwarderSshCmd(fmt.Sprintf("echo %s | sudo tee -a /var/vcap/sys/log/other-job/app.log", message))
+		Eventually(session).Should(gexec.Exit(0))
+		return ForwarderLog()
+	}
+}
+
 func WriteToPrivateTestFile(message string) func() *gexec.Session {
 	return func() *gexec.Session {
 		session := ForwarderSshCmd(fmt.Sprintf("sudo bash -c '"+

tests/manifests/blackbox-unpriv.yml

@@ -3,6 +3,8 @@ name: ((deployment))
 releases:
   - name: syslog
     version: latest
+  - name: bpm
+    version: latest
 stemcells:
   - alias: default
     os: ((stemcell-os))
@@ -17,6 +19,8 @@ instance_groups:
     azs:
       - z1
     jobs:
+      - name: bpm
+        release: bpm
       - name: syslog_forwarder
         release: syslog
         properties:

tests/manifests/broken-rules.yml

@@ -3,6 +3,8 @@ name: ((deployment))
 releases:
   - name: syslog
     version: latest
+  - name: bpm
+    version: latest
 stemcells:
   - alias: default
     os: ((stemcell-os))
@@ -17,6 +19,8 @@ instance_groups:
     azs:
       - z1
     jobs:
+      - name: bpm
+        release: bpm
       - name: syslog_forwarder
         release: syslog
     properties:

tests/manifests/debug-filtering.yml

@@ -3,6 +3,8 @@ name: ((deployment))
 releases:
   - name: syslog
     version: latest
+  - name: bpm
+    version: latest
 stemcells:
   - alias: default
     os: ((stemcell-os))
@@ -17,6 +19,8 @@ instance_groups:
     azs:
       - z1
     jobs:
+      - name: bpm
+        release: bpm
       - name: syslog_forwarder
         release: syslog
     properties:

tests/manifests/disabled-no-config.yml

@@ -3,6 +3,8 @@ name: ((deployment))
 releases:
   - name: syslog
     version: latest
+  - name: bpm
+    version: latest
 stemcells:
   - alias: default
     os: ((stemcell-os))
@@ -17,6 +19,8 @@ instance_groups:
     azs:
       - z1
     jobs:
+      - name: bpm
+        release: bpm
       - name: syslog_forwarder
         release: syslog
     properties:

tests/manifests/disabled.yml

@@ -3,6 +3,8 @@ name: ((deployment))
 releases:
   - name: syslog
     version: latest
+  - name: bpm
+    version: latest
 stemcells:
   - alias: default
     os: ((stemcell-os))
@@ -17,6 +19,8 @@ instance_groups:
     azs:
       - z1
     jobs:
+      - name: bpm
+        release: bpm
       - name: syslog_forwarder
         release: syslog
     properties:

tests/manifests/environment-identifier.yml

@@ -3,6 +3,8 @@ name: ((deployment))
 releases:
   - name: syslog
     version: latest
+  - name: bpm
+    version: latest
 stemcells:
   - alias: default
     os: ((stemcell-os))
@@ -17,6 +19,8 @@ instance_groups:
     azs:
       - z1
     jobs:
+      - name: bpm
+        release: bpm
       - name: syslog_forwarder
         release: syslog
     properties:

tests/manifests/good-rules.yml

@@ -3,6 +3,8 @@ name: ((deployment))
 releases:
   - name: syslog
     version: latest
+  - name: bpm
+    version: latest
 stemcells:
   - alias: default
     os: ((stemcell-os))
@@ -17,6 +19,8 @@ instance_groups:
     azs:
       - z1
     jobs:
+      - name: bpm
+        release: bpm
       - name: syslog_forwarder
         release: syslog
     properties:

tests/manifests/relp-tls.yml

@@ -3,6 +3,8 @@ name: ((deployment))
 releases:
   - name: syslog
     version: latest
+  - name: bpm
+    version: latest
 stemcells:
   - alias: default
     os: ((stemcell-os))
@@ -17,6 +19,8 @@ instance_groups:
     azs:
       - z1
     jobs:
+      - name: bpm
+        release: bpm
       - name: syslog_forwarder
         release: syslog
         properties:

tests/manifests/tcp-blackbox.yml

@@ -3,6 +3,8 @@ name: ((deployment))
 releases:
   - name: syslog
     version: latest
+  - name: bpm
+    version: latest
 stemcells:
   - alias: default
     os: ((stemcell-os))
@@ -17,6 +19,8 @@ instance_groups:
     azs:
       - z1
     jobs:
+      - name: bpm
+        release: bpm
       - name: syslog_forwarder
         release: syslog
     properties:

tests/manifests/tls-forwarding-mtls.yml

@@ -3,6 +3,8 @@ name: ((deployment))
 releases:
   - name: syslog
     version: latest
+  - name: bpm
+    version: latest
 stemcells:
   - alias: default
     os: ((stemcell-os))
@@ -17,6 +19,8 @@ instance_groups:
     azs:
       - z1
     jobs:
+      - name: bpm
+        release: bpm
       - name: syslog_forwarder
         release: syslog
         properties:

tests/manifests/tls-forwarding.yml

@@ -3,6 +3,8 @@ name: ((deployment))
 releases:
   - name: syslog
     version: latest
+  - name: bpm
+    version: latest
 stemcells:
   - alias: default
     os: ((stemcell-os))
@@ -17,6 +19,8 @@ instance_groups:
     azs:
       - z1
     jobs:
+      - name: bpm
+        release: bpm
       - name: syslog_forwarder
         release: syslog
         properties:

tests/manifests/udp-blackbox.yml

@@ -3,6 +3,8 @@ name: ((deployment))
 releases:
   - name: syslog
     version: latest
+  - name: bpm
+    version: latest
 stemcells:
   - alias: default
     os: ((stemcell-os))
@@ -17,6 +19,8 @@ instance_groups:
     azs:
       - z1
     jobs:
+      - name: bpm
+        release: bpm
       - name: syslog_forwarder
         release: syslog
   - name: storer

tests/manifests/vcap-filtering.yml

@@ -3,6 +3,8 @@ name: ((deployment))
 releases:
   - name: syslog
     version: latest
+  - name: bpm
+    version: latest
 stemcells:
   - alias: default
     os: ((stemcell-os))
@@ -17,6 +19,8 @@ instance_groups:
     azs:
       - z1
     jobs:
+      - name: bpm
+        release: bpm
       - name: syslog_forwarder
         release: syslog
     properties: