Retry resize2fs on transient Permission denied during persistent disk grow

platform/disk/linux_formatter.go

@@ -3,20 +3,34 @@ package disk
 import (
 	"regexp"
 	"strings"
+	"time"
 
+	"code.cloudfoundry.org/clock"
 	bosherr "github.com/cloudfoundry/bosh-utils/errors"
 	boshsys "github.com/cloudfoundry/bosh-utils/system"
 )
 
+const (
+	growFilesystemRetryDelay    = 5 * time.Second
+	growFilesystemMaxRetries    = 10
+	growFilesystemPermDeniedMsg = "Permission denied to resize filesystem"
+)
+
 type linuxFormatter struct {
-	runner boshsys.CmdRunner
-	fs     boshsys.FileSystem
+	runner      boshsys.CmdRunner
+	fs          boshsys.FileSystem
+	timeService clock.Clock
 }
 
 func NewLinuxFormatter(runner boshsys.CmdRunner, fs boshsys.FileSystem) Formatter {
+	return NewLinuxFormatterWithClock(runner, fs, clock.NewClock())
+}
+
+func NewLinuxFormatterWithClock(runner boshsys.CmdRunner, fs boshsys.FileSystem, timeService clock.Clock) Formatter {
 	return linuxFormatter{
-		runner: runner,
-		fs:     fs,
+		runner:      runner,
+		fs:          fs,
+		timeService: timeService,
 	}
 }
 
@@ -74,11 +88,7 @@ func (f linuxFormatter) GrowFilesystem(partitionPath string) error {
 
 	switch existingFsType {
 	case FileSystemExt4:
-		_, _, _, err := f.runner.RunCommand(
-			"resize2fs",
-			"-f",
-			partitionPath,
-		)
+		err = f.growExt4WithRetry(partitionPath)
 		if err != nil {
 			return bosherr.WrapError(err, "Failed to grow Ext4 filesystem")
 		}
@@ -97,6 +107,27 @@ func (f linuxFormatter) GrowFilesystem(partitionPath string) error {
 	return nil
 }
 
+// growExt4WithRetry retries resize2fs when the block device reports "Permission
+// denied". This can happen transiently after an online volume extension while
+// the block device's new size is not yet consistently visible to the guest, and
+// resolves once the underlying storage layer settles.
+func (f linuxFormatter) growExt4WithRetry(partitionPath string) error {
+	var err error
+	for retry := range growFilesystemMaxRetries + 1 {
+		if retry > 0 {
+			f.timeService.Sleep(growFilesystemRetryDelay)
+		}
+		_, _, _, err = f.runner.RunCommand("resize2fs", "-f", partitionPath)
+		if err == nil {
+			return nil
+		}
+		if !strings.Contains(err.Error(), growFilesystemPermDeniedMsg) {
+			return err
+		}
+	}
+	return err
+}
+
 func (f linuxFormatter) makeFileSystemExt4(partitionPath string) error {
 	var err error
 	if f.fs.FileExists("/sys/fs/ext4/features/lazy_itable_init") {

platform/disk/linux_formatter_test.go

@@ -9,6 +9,7 @@ import (
 
 	fakesys "github.com/cloudfoundry/bosh-utils/system/fakes"
 
+	fakeboshaction "github.com/cloudfoundry/bosh-agent/v2/agent/action/fakes"
 	. "github.com/cloudfoundry/bosh-agent/v2/platform/disk"
 )
 
@@ -228,18 +229,20 @@ var _ = Describe("Linux Formatter", func() {
 		var (
 			fakeRunner *fakesys.FakeCmdRunner
 			fakeFs     *fakesys.FakeFileSystem
+			fakeClock  *fakeboshaction.FakeClock
 			formatter  Formatter
 		)
 
 		BeforeEach(func() {
 			fakeRunner = fakesys.NewFakeCmdRunner()
 			fakeFs = fakesys.NewFakeFileSystem()
+			fakeClock = &fakeboshaction.FakeClock{}
 		})
 
 		Context("when determining partition filesystem fails", func() {
 			BeforeEach(func() {
 				fakeRunner.AddCmdResult("blkid -p /dev/nvme2n1p1", fakesys.FakeCmdResult{ExitStatus: 1, Error: errors.New("No GPT found")})
-				formatter = NewLinuxFormatter(fakeRunner, fakeFs)
+				formatter = NewLinuxFormatterWithClock(fakeRunner, fakeFs, fakeClock)
 			})
 
 			It("returns an error", func() {
@@ -253,7 +256,7 @@ var _ = Describe("Linux Formatter", func() {
 		Context("when using Ext4", func() {
 			BeforeEach(func() {
 				fakeRunner.AddCmdResult("blkid -p /dev/nvme2n1p1", fakesys.FakeCmdResult{Stdout: `xxxxx TYPE="ext4" yyyy zzzz`})
-				formatter = NewLinuxFormatter(fakeRunner, fakeFs)
+				formatter = NewLinuxFormatterWithClock(fakeRunner, fakeFs, fakeClock)
 			})
 
 			It("grows the Ext4 filesystem", func() {
@@ -263,25 +266,60 @@ var _ = Describe("Linux Formatter", func() {
 				Expect(fakeRunner.RunCommands[1]).To(Equal([]string{"resize2fs", "-f", "/dev/nvme2n1p1"}))
 			})
 
-			Context("when resize2fs fails", func() {
+			Context("when resize2fs fails with a non-retryable error", func() {
 				BeforeEach(func() {
 					fakeRunner.AddCmdResult("resize2fs -f /dev/nvme2n1p1", fakesys.FakeCmdResult{ExitStatus: 1, Error: errors.New("resize2fs failure")})
 				})
 
-				It("returns an error", func() {
+				It("returns the error immediately without retrying", func() {
 					err := formatter.GrowFilesystem("/dev/nvme2n1p1")
 
 					Expect(err).To(HaveOccurred())
 					Expect(err.Error()).To(ContainSubstring("Failed to grow Ext4 filesystem"))
 					Expect(err.Error()).To(ContainSubstring("resize2fs failure"))
+					Expect(fakeClock.SleepCallCount()).To(Equal(0))
+				})
+			})
+
+			Context("when resize2fs fails with 'Permission denied to resize filesystem'", func() {
+				BeforeEach(func() {
+					permDeniedErr := errors.New("Permission denied to resize filesystem")
+					fakeRunner.AddCmdResult("resize2fs -f /dev/nvme2n1p1", fakesys.FakeCmdResult{ExitStatus: 1, Error: permDeniedErr})
+					fakeRunner.AddCmdResult("resize2fs -f /dev/nvme2n1p1", fakesys.FakeCmdResult{ExitStatus: 1, Error: permDeniedErr})
+					fakeRunner.AddCmdResult("resize2fs -f /dev/nvme2n1p1", fakesys.FakeCmdResult{ExitStatus: 0})
+				})
+
+				It("retries until success and sleeps between attempts", func() {
+					err := formatter.GrowFilesystem("/dev/nvme2n1p1")
+
+					Expect(err).NotTo(HaveOccurred())
+					Expect(fakeClock.SleepCallCount()).To(Equal(2))
+				})
+			})
+
+			Context("when resize2fs keeps failing with 'Permission denied' for all attempts", func() {
+				BeforeEach(func() {
+					permDeniedErr := errors.New("Permission denied to resize filesystem")
+					for i := 0; i < 11; i++ {
+						fakeRunner.AddCmdResult("resize2fs -f /dev/nvme2n1p1", fakesys.FakeCmdResult{ExitStatus: 1, Error: permDeniedErr})
+					}
+				})
+
+				It("returns an error after exhausting all attempts", func() {
+					err := formatter.GrowFilesystem("/dev/nvme2n1p1")
+
+					Expect(err).To(HaveOccurred())
+					Expect(err.Error()).To(ContainSubstring("Failed to grow Ext4 filesystem"))
+					Expect(err.Error()).To(ContainSubstring("Permission denied to resize filesystem"))
+					Expect(fakeClock.SleepCallCount()).To(Equal(10))
 				})
 			})
 		})
 
 		Context("when using XFS", func() {
 			BeforeEach(func() {
 				fakeRunner.AddCmdResult("blkid -p /dev/nvme2n1p1", fakesys.FakeCmdResult{Stdout: `xxxxx TYPE="xfs" yyyy zzzz`})
-				formatter = NewLinuxFormatter(fakeRunner, fakeFs)
+				formatter = NewLinuxFormatterWithClock(fakeRunner, fakeFs, fakeClock)
 			})
 
 			It("grows the XFS filesystem", func() {