-
Notifications
You must be signed in to change notification settings - Fork 3
Expand file tree
/
Copy pathplan_flush.go
More file actions
1814 lines (1717 loc) · 80.5 KB
/
Copy pathplan_flush.go
File metadata and controls
1814 lines (1717 loc) · 80.5 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
// SPDX-License-Identifier: Apache-2.0
package git
import (
"bytes"
"context"
"errors"
"fmt"
"math"
"os"
"path"
"path/filepath"
"sort"
"strings"
gogit "github.com/go-git/go-git/v6"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"sigs.k8s.io/controller-runtime/pkg/log"
sigsyaml "sigs.k8s.io/yaml"
v1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3"
"github.com/ConfigButler/gitops-reverser/internal/git/manifestedit"
"github.com/ConfigButler/gitops-reverser/internal/manifestanalyzer"
"github.com/ConfigButler/gitops-reverser/internal/manifestreport"
"github.com/ConfigButler/gitops-reverser/internal/types"
"github.com/ConfigButler/gitops-reverser/internal/typeset"
)
// flushEventsToWorktree is the plan-then-flush write path (M7), described in
// docs/spec/current-manifest-support-review.md ("Writer Model: Plan,
// Apply, Dirty Flush"). It replaces the per-event locate+write loop: it builds the
// byte-free structure model for the GitTarget subtree once, resolves each coalesced
// event to a single-identity action over that model, applies the actions to
// hydrated commit-scoped file buffers, and flushes only the files whose bytes
// changed or were deleted. It returns true when at least one file was written or
// removed.
//
// This is the steady-state half of the design's "Two Paths, One Plan Type"
// (docs/spec/reconcile-via-watchlist-mark-and-sweep.md): every event is
// a single-identity intent — an upsert (create/patch/replace) for an object-bearing
// event, or a delete-document for a DELETE — and the writer NEVER mark-and-sweeps a
// batch. Whole-folder mark-and-sweep is the resync mechanism (M8), not steady state.
// mapperForCluster returns the GVK->GVR lookup for a source cluster: the per-cluster registry
// when a cluster is named and a cluster resolver is wired, else the default (local) mapper.
// The CLI and tests leave clusterMapper nil, so they always resolve against `mapper`.
func (w *BranchWorker) mapperForCluster(clusterID string) typeset.Lookup {
if clusterID != "" && w.clusterMapper != nil {
if lk := w.clusterMapper(clusterID); lk != nil {
return lk
}
}
return w.mapper
}
// clusterIDForEvents returns the source cluster the events in one base belong to. Events in a
// single flush share a GitTarget (they are grouped by base), so they share a cluster; the
// first non-empty id wins, and an all-empty set is the local cluster.
func clusterIDForEvents(events []Event) string {
for _, ev := range events {
if ev.SourceCluster != "" {
return ev.SourceCluster
}
}
return ""
}
func (w *BranchWorker) flushEventsToWorktree(
ctx context.Context,
worktree *gogit.Worktree,
base string,
events []Event,
policy *manifestanalyzer.PlacementPolicy,
pruneMode v1alpha3.PruneMode,
) (bool, error) {
root := worktree.Filesystem().Root()
scoped, err := scanRenderScope(root, base)
if err != nil {
return false, err
}
// Every event in a base shares one GitTarget (events are grouped by base), so they share
// one source cluster; resolve this subtree's GVK->GVR against that cluster's registry.
mapper := w.mapperForCluster(clusterIDForEvents(events))
batch := newWriteBatch(ctx, w.contentWriter, mapper, scoped.scan, policy, scoped.writeSubdir)
batch.pruneMode = pruneMode
batch.target = placementTargetForEvents(events)
if err := batch.refusal(); err != nil {
return false, err
}
for _, event := range events {
if err := batch.applyEvent(ctx, event); err != nil {
return false, err
}
}
// The flush is anchored at renderBase — spec.path, or the common ancestor of spec.path
// and every base it reads. The write jail (writeSubdir) is enforced inside the batch, so
// a planned write outside spec.path is refused even though the scan reached past it.
return batch.flush(ctx, worktree, root, scoped.renderBase)
}
// writeBatch is the commit-scoped plan-then-flush working set for one GitTarget
// subtree. The store is the byte-free model the batch resolves identities against;
// contentByPath holds the worktree bytes so a touched file is hydrated lazily into
// a fileBuffer; buffers accumulates the mutations the events produce.
type writeBatch struct {
writer eventContentWriter
mapper typeset.Lookup
store *manifestanalyzer.ManifestStore
docLoc map[*manifestanalyzer.DocumentModel]manifestanalyzer.RecordRef
contentByPath map[string][]byte
buffers map[string]*fileBuffer
// intents records what each document this flush writes must render to, so the
// render precondition can tell a change the flush MEANT from one it merely caused.
// Anything not named here has to come out of the re-render untouched.
intents []manifestanalyzer.WriteIntent
// putToKustomize records that this flush touched a kustomize render root — it edited a
// governed document, or placed a new one into a kustomization's resources:. It is what
// turns the oracle on, and it is deliberately NOT the same question as WriteIntent.Governed:
// that one additionally ASSERTS the document is rendered, which a new document is not
// entitled to claim (its resources: entry can legitimately fail to be added — see
// appendKustomizationResource — leaving the file written but outside every render).
putToKustomize bool
// target is the GitTarget this batch writes for, carried only so the placement metrics
// can name it (see placement_metrics.go). It is set by the caller — the live path reads
// it off the events, the resync path from the request — and is empty for the CLI and for
// tests, where the counters are simply unlabelled.
target placementTarget
// policy is the GitTarget's declared new-file placement policy, consulted
// only for a resource with no existing document. nil means no declared policy —
// placement falls through to the folder's one kustomize root and then the canonical path.
policy *manifestanalyzer.PlacementPolicy
// pruneMode is the GitTarget's effective spec.prune.mode, gating the EXPLICIT delete
// path only (applyDelete). The inferred mark-and-sweep is gated a layer up, in the
// planner, so a suppressed drop never becomes an action in the first place.
//
// Set only on the live-event batch, because that is the only batch that folds DELETE
// events; the resync batch drops documents through the plan instead and leaves this
// zero. It is therefore always read through OrDefault: the zero value is unset, not
// `never`, and reading it literally would make a batch that simply never set it stop
// mirroring deletes.
pruneMode v1alpha3.PruneMode
// writeSubdir is spec.path expressed relative to the render anchor (renderBase) — the
// write jail. It is "" for a self-contained subtree (renderBase == spec.path), where
// every scanned path is writable; it is non-empty only when the scan reached past
// spec.path into a base it renders (render-root scoping), and then a planned write must
// stay within it. The store and every path in it are keyed relative to renderBase, so a
// writable path is one under writeSubdir. See internal/git/render_scope.go.
writeSubdir string
// coldBundles tracks, per path, the new resources this batch has placed at a
// path that held no document before the batch started (keyed the same as
// buffers). It exists so several new resources that render to the same
// brand-new path — a collision LocateNew resolves against the pre-batch store
// and therefore cannot see coming — form one deterministic, resource-identity-
// sorted multi-document file instead of each writeWholeFile call silently
// discarding the one before it. See
// docs/spec/gittarget-new-file-placement-rules.md,
// "Collision and append behavior": "if several new plaintext resources in one
// plan render to the same path, write a multi-document file in deterministic
// resource-identity order."
coldBundles map[string][]coldBundleMember
}
// coldBundleMember is one new document contributing to a brand-new shared bundle
// file within this batch. Retained (rather than re-parsed from buf.current) so a
// later collision on the same path can re-sort and rebuild the whole file from
// scratch, independent of which new resource's event the writer processed first.
type coldBundleMember struct {
identifier types.ResourceIdentifier
content []byte
// sensitive records whether this member is an encrypted (sensitive) resource, so
// createNew can refuse to co-mingle sensitive and plaintext documents in one
// brand-new file regardless of the order their events arrived (Option B2's
// write-safety guard — see createNew).
sensitive bool
}
func newWriteBatch(
ctx context.Context,
writer eventContentWriter,
mapper typeset.Lookup,
scan manifestanalyzer.FolderScan,
policy *manifestanalyzer.PlacementPolicy,
writeSubdir string,
) *writeBatch {
// The writer allowlist retains build directives (kustomization.yaml) and the operator's
// own .sops.yaml bootstrap config outside the managed model — these are auxiliary input,
// not documents to materialise or to mis-refuse as standalone non-KRM. Every other KRM
// document is still materialised: the live writer indexes the whole subtree for
// placement. The scan also carries the foreign-content view and the active
// .gittargetignore, so the structure-only acceptance gate (run by writeBatch.refusal) and
// the write-plan precondition (run by writeBatch.flush) read both from the store.
store := manifestanalyzer.BuildStoreFromScan(ctx, scan, mapper, manifestanalyzer.WriterAllowlist())
// Surface the store's build-time warnings (ambiguous namespace or override
// context, scope mismatches) once per batch: these drive silent fallbacks —
// e.g. an ambiguous override chain falls back to write-through — and without
// this line the live path would leave no trace of why. The analyzer CLI and
// scan mode show the same diagnostics offline.
logStoreDiagnostics(ctx, store.Diagnostics)
contentByPath := make(map[string][]byte, len(scan.YAMLFiles))
for _, f := range scan.YAMLFiles {
contentByPath[f.Path] = f.Content
}
return &writeBatch{
writer: writer,
mapper: mapper,
store: store,
docLoc: store.DocumentLocations(),
contentByPath: contentByPath,
buffers: map[string]*fileBuffer{},
policy: policy,
writeSubdir: writeSubdir,
}
}
// refusal runs the structure-only acceptance gate over the batch's store and returns a
// *manifestanalyzer.AcceptanceRefusedError when the GitTarget subtree holds content the
// operator cannot safely manage: a duplicate manifest identity, an impure managed file, a
// standalone non-KRM / invalid YAML file, a managed resource hiding in a build directive,
// or an unsupported kustomization. A refusal aborts the commit before any file is touched,
// so the folder is left exactly as the human left it until they clean it.
//
// It is structure-only on purpose: the writer must never refuse on a discovery-derived
// followability fact (unwatched / out-of-scope), which can blink on a discovery wobble and
// would otherwise turn a transient into a stuck, unwritable GitTarget.
func (wb *writeBatch) refusal() error {
return manifestanalyzer.RefusalError(manifestanalyzer.AcceptStructureOnly(wb.store))
}
// fileBuffer is the commit-scoped, hydrated working copy of one file under the
// GitTarget base path. original is the worktree bytes (nil for a file the batch
// creates); current is the bytes after applying actions (nil means the file should
// be removed). Dirty/Deleted are derived exactly as the design's FileModel — two
// byte slices are the whole state machine, so there is no flag to forget to flip.
type fileBuffer struct {
rel string
original []byte
current []byte
}
func (b *fileBuffer) dirty() bool { return b.current != nil && !bytes.Equal(b.current, b.original) }
func (b *fileBuffer) deleted() bool { return b.current == nil && b.original != nil }
// buffer returns the hydrated working copy for a base-relative path, reading the
// worktree bytes into Original/Current on first touch. A path with no worktree
// bytes is a new file (Original nil).
func (wb *writeBatch) buffer(rel string) *fileBuffer {
if b, ok := wb.buffers[rel]; ok {
return b
}
b := &fileBuffer{rel: rel}
if orig, ok := wb.contentByPath[rel]; ok {
b.original = orig
b.current = orig
}
wb.buffers[rel] = b
return b
}
// upsertOutcome is what an upsert actually did to the worktree bytes, so a caller can
// count create/update accurately from the apply rather than from a separate plan
// estimate (which mislabels a re-encrypted sensitive resource as skipped).
type upsertOutcome int
const (
upsertNoChange upsertOutcome = iota
upsertCreated
upsertUpdated
// upsertSkippedUnsafe is a deliberate, fail-safe refusal to write a resource:
// its placement could not be resolved safely, or writing would co-mingle a
// sensitive and a plaintext document, or would overwrite a multi-document file.
// It is distinct from upsertNoChange (a genuine no-op) so the resync path can
// count it and surface it, rather than have a not-mirrored resource vanish with
// no signal (placement Option B2's fail-safe skips — see createNew/writeWholeFile).
upsertSkippedUnsafe
)
// applyEvent folds one event into the batch: a field patch sets bounded fields on an
// existing parent, a DELETE removes a document, anything else is an upsert (the
// object-bearing event the stream guarantees for non-deletes). The steady-state
// writer does not need the upsert outcome (it flushes by byte state), so it is
// discarded here; the resync planner consumes it for stats.
func (wb *writeBatch) applyEvent(ctx context.Context, event Event) error {
switch {
case event.IsFieldPatch():
return wb.applyFieldPatch(ctx, event)
case event.Operation == "DELETE":
wb.applyDelete(ctx, event)
return nil
default:
_, err := wb.applyUpsert(ctx, event)
return err
}
}
// applyUpsert resolves an object-bearing event against the subtree. When a managed
// document for its identity already lives there — even moved off the canonical path —
// the resource is edited where it lives: a non-sensitive document is patched in place;
// a sensitive document is re-encrypted wholesale AT ITS EXISTING PATH (never patched in
// place — that would drop the SOPS metadata and write the secret back in cleartext, and
// never at the canonical path, which would orphan the moved copy). A resource with no
// existing document is placed by createNew. It returns what it did to the bytes
// (created / updated / no change).
func (wb *writeBatch) applyUpsert(ctx context.Context, event Event) (upsertOutcome, error) {
id, ok := manifestIdentity(event.Object)
if !ok {
return wb.createNew(ctx, event)
}
dm := wb.store.ByManifestIdentity[id]
if dm == nil {
return wb.createNew(ctx, event)
}
filePath := wb.docLoc[dm].FilePath
if !wb.writer.isSensitiveIdentifier(event.Identifier) {
return wb.patchExisting(ctx, event, filePath, id, dm)
}
return wb.rewriteSensitive(ctx, event, filePath)
}
// rewriteSensitive re-encrypts a sensitive document wholesale at its existing path.
//
// Its intent is UNCHECKED: the file is SOPS ciphertext, so kustomize renders the encrypted
// blob and no plaintext live object can ever equal it. The oracle is told to expect this
// object to move without being able to say what to — while still holding the write to
// disturbing nothing else, which is the half that protects other environments.
func (wb *writeBatch) rewriteSensitive(ctx context.Context, event Event, filePath string) (upsertOutcome, error) {
outcome, err := wb.writeWholeFile(ctx, event, filePath)
if err == nil && wroteBytes(outcome) {
wb.intend(markUnchecked(intentFor(event.Object, filePath, false), true))
}
return outcome, err
}
// wroteBytes reports whether an upsert actually changed the worktree, which is the only
// case that owes the oracle an intent.
func wroteBytes(o upsertOutcome) bool {
return o == upsertCreated || o == upsertUpdated
}
// createNew resolves the placement of a resource with no existing document —
// declared policy (Option B), the folder's one kustomize root, or the canonical
// fallback — per docs/spec/gittarget-new-file-placement-rules.md,
// adds the kustomize resources: entry the placement may require, and writes the new
// document: a brand-new file, or an additional document appended to an existing
// accepted plaintext bundle. A placement LocateNew cannot honour safely (today, only
// a sensitive resource whose resolved path collides with an existing file) is logged
// and left unwritten rather than risking a mis-write; the next event or resync
// retries it once the conflict is resolved (e.g. the placement policy is fixed).
func (wb *writeBatch) createNew(ctx context.Context, event Event) (upsertOutcome, error) {
kind := ""
if event.Object != nil {
kind = event.Object.GetKind()
}
sensitive := wb.writer.isSensitiveIdentifier(event.Identifier)
// WriteScope tells placement the write jail: when render-root scoping re-rooted the scan
// past spec.path, a declared/canonical path is rebased under the jail rather than escaping
// it (see finishPlacement). It is "" for a self-contained subtree, where placement already
// resolves relative to spec.path.
placement, err := manifestanalyzer.LocateNew(wb.store, wb.policy, manifestanalyzer.PlacementRequest{
Identifier: event.Identifier,
Kind: kind,
Sensitive: sensitive,
WriteScope: wb.writeSubdir,
})
if err != nil {
refusal := placementRefusalReason(err)
log.FromContext(ctx).Info("Skipping new resource: placement could not be resolved safely",
"resource", event.Identifier.String(), "refusal", string(refusal), "reason", err.Error())
recordPlacementRefusal(ctx, wb.target, event.Identifier, refusal)
return upsertSkippedUnsafe, nil
}
// The LIVE object, kept before the namespace strip below rewrites it. The bytes we write
// and the object the render must produce are not the same thing, and only this scope
// still holds both — see intentFor.
live := event.Object
// A destination that infers its namespace from build context (a kustomization's
// namespace: transformer) must keep metadata.namespace out of the written bytes,
// exactly as patchExisting already does for an in-place edit of an existing
// document in the same context — otherwise the new document would silently break
// the convention every sibling in that directory follows.
if placement.NamespaceInherited && event.Object != nil {
event.Object = event.Object.DeepCopy()
event.Object.SetNamespace("")
}
outcome, refusal, err := wb.placeNewDocument(ctx, event, placement, sensitive)
if err != nil || !wroteBytes(outcome) {
// A skipped write is a resource the mirror does not hold. Count it with the refusals
// LocateNew raised, from the same closed reason set, so "resources we declined to
// place" is one series rather than a log line here and a metric there.
if outcome == upsertSkippedUnsafe {
recordPlacementRefusal(ctx, wb.target, event.Identifier, refusal)
}
return outcome, err
}
// Recorded here rather than at resolution: this is the point at which the document is
// really in the mirror at this path, so placements_total and placement_refusals_total
// partition every new resource instead of double-counting the ones that resolved and
// then could not be written.
recordPlacement(ctx, wb.target, event.Identifier, placement.Source, placement.Append)
// AFTER the write, for the same reason the placement is counted here. placeNewDocument can
// still decline — a multi-document target it will not overwrite, or a new file that would
// mix sensitive and plaintext documents — and registering the entry first meant a resource we
// REFUSED still put its file into the folder's render. For the multi-document case that is
// foreign content we declined to own, added to resources: on our say-so; and either way it
// counted as outcome="added", the value that is supposed to mean "the file we just wrote will
// build". Pinned by TestPlacementMetrics_RefusedPlacementLeavesTheKustomizationAlone.
if placement.Kustomization != nil {
wb.appendKustomizationResource(ctx, event, placement)
}
// A new document that joins a kustomization's resources: list is INSIDE a render root, so
// the folder's images:/replicas: entries govern it from the moment it lands — and we do not
// route a new document's values onto an entry (it has no override chain yet; it did not
// exist when the store was built). So the live value goes into the file, and if an entry
// overrides it, the folder renders something else and the resource never converges.
//
// Declaring it governed puts it in front of the oracle, which turns that from a silent
// non-converging commit into a reported refusal naming the file and the object. It does not
// make the write work — that needs attribution for a document that does not exist yet — but
// "we cannot express this here" is an answer, and quietly writing a lie is not.
wb.putToKustomize = wb.putToKustomize || placement.Kustomization != nil
wb.intend(markUnchecked(intentFor(live, placement.Path, false), sensitive))
return outcome, nil
}
// placeNewDocument writes the new document at its resolved placement: appended to an existing
// accepted bundle, folded into a same-batch cold bundle, or as a file of its own.
//
// It returns the refusal reason alongside the outcome, and only for upsertSkippedUnsafe, so
// the caller can count WHY a new resource was left out of the mirror without inspecting a log
// message. The multi-document refusal is attributed here rather than inside writeWholeFile
// because that function also serves in-place updates, where the same skip is not a placement
// decision at all.
func (wb *writeBatch) placeNewDocument(
ctx context.Context,
event Event,
placement manifestanalyzer.PlacementResult,
sensitive bool,
) (upsertOutcome, manifestanalyzer.PlacementRefusalReason, error) {
if placement.Append {
outcome, err := wb.appendNewDocument(ctx, event, placement.Path)
return outcome, "", err
}
buf := wb.buffer(placement.Path)
if buf.original == nil {
// Nothing occupied this path before the batch started, so every write
// here is a new resource: this event, or an earlier one in the same
// batch that rendered to the same path (a collision LocateNew cannot
// see coming — it only ever consults the pre-batch store). Route
// through the cold-bundle path so a collision forms a deterministic
// multi-document file instead of a second writeWholeFile silently
// discarding whichever new resource arrived first.
//
// A sensitive resource must never share a file (with anything), and a
// plaintext resource must never join a bundle that already holds a
// sensitive member — either way the file would co-mingle encrypted and
// plaintext documents. Skip rather than mix; the next event or resync
// retries once the placement policy stops routing them together. This is
// the same-batch half of Option B2's write-safety guard (the cross-batch
// half — appending into an already-encrypted file — is refused in
// LocateNew/finishPlacement).
if buf.current != nil && (sensitive || wb.coldBundleHasSensitive(placement.Path)) {
log.FromContext(ctx).Info(
"Skipping new resource: sensitive and plaintext resources must not share a new file",
"resource", event.Identifier.String(), "file", placement.Path, "sensitive", sensitive)
return upsertSkippedUnsafe, manifestanalyzer.PlacementRefusedMixedSensitivityNewFile, nil
}
outcome, err := wb.writeColdBundleMember(ctx, event, placement.Path, sensitive)
return outcome, "", err
}
outcome, err := wb.writeWholeFile(ctx, event, placement.Path)
if outcome == upsertSkippedUnsafe {
return outcome, manifestanalyzer.PlacementRefusedMultiDocumentTarget, err
}
return outcome, "", err
}
// writeColdBundleMember writes a resource with no existing document to rel, a
// path nothing occupied before this batch started. Because LocateNew resolves
// every event against the pre-batch store snapshot (P2 of the design doc),
// several new resources rendering to the same brand-new path each look like the
// sole occupant to LocateNew, so a plain single-document write would let each
// one overwrite the last. Instead every member seen so far at rel (including
// this one) is re-sorted by resource identity and the file is rebuilt from
// scratch, so the result is independent of which new resource's event the
// writer processed first — see the design doc's "Collision and append
// behavior": "if several new plaintext resources in one plan render to the same
// path, write a multi-document file in deterministic resource-identity order."
// For the common single-member case this produces byte-identical output to a
// plain write.
func (wb *writeBatch) writeColdBundleMember(
ctx context.Context,
event Event,
rel string,
sensitive bool,
) (upsertOutcome, error) {
content, err := wb.writer.buildContentForWrite(ctx, event)
if err != nil {
return upsertNoChange, err
}
if wb.coldBundles == nil {
wb.coldBundles = map[string][]coldBundleMember{}
}
wb.coldBundles[rel] = append(
wb.coldBundles[rel],
coldBundleMember{identifier: event.Identifier, content: content, sensitive: sensitive},
)
members := wb.coldBundles[rel]
sort.Slice(members, func(i, j int) bool {
return members[i].identifier.Key() < members[j].identifier.Key()
})
var rebuilt []byte
for _, m := range members {
rebuilt = appendYAMLDocument(rebuilt, m.content)
}
wb.buffer(rel).current = rebuilt
return upsertCreated, nil
}
// coldBundleHasSensitive reports whether any member already staged for the
// brand-new file at rel is an encrypted (sensitive) resource, so createNew can
// refuse to add a plaintext member that would co-mingle with it.
func (wb *writeBatch) coldBundleHasSensitive(rel string) bool {
for _, m := range wb.coldBundles[rel] {
if m.sensitive {
return true
}
}
return false
}
// appendNewDocument adds a resource with no existing document as an additional
// document in an existing accepted plaintext file (a "bundle" placement). Unlike
// writeWholeFile it never replaces the file's existing bytes — every prior document
// in the buffer survives untouched, byte for byte; LocateNew never returns an
// Append placement for a sensitive resource (see its doc comment), so this path is
// plaintext-only.
func (wb *writeBatch) appendNewDocument(ctx context.Context, event Event, rel string) (upsertOutcome, error) {
content, err := wb.writer.buildContentForWrite(ctx, event)
if err != nil {
return upsertNoChange, err
}
buf := wb.buffer(rel)
buf.current = appendYAMLDocument(buf.current, content)
return upsertCreated, nil
}
// appendYAMLDocument appends newDoc as an additional "---\n"-separated document
// after existing. existing is assumed to already be valid, accepted YAML (single- or
// multi-document); newDoc is assumed to be exactly one well-formed document
// (sanitize.MarshalToOrderedYAML's output, which always ends in a newline).
func appendYAMLDocument(existing, newDoc []byte) []byte {
if len(existing) == 0 {
return newDoc
}
const separator = "---\n"
out := make([]byte, 0, len(existing)+len(separator)+len(newDoc))
out = append(out, existing...)
if out[len(out)-1] != '\n' {
out = append(out, '\n')
}
out = append(out, separator...)
out = append(out, newDoc...)
return out
}
// appendKustomizationResource adds the new document's path to its resources:
// sequence as part of the same commit, so kustomize picks up the file createNew just
// placed inside the kustomization's directory — the "add to the right kustomize
// file." The entry is rendered relative to the kustomization's own directory
// (resources: entries are relative to the kustomization file, not the repo root).
// A failure here only drops the resources: entry (logged as a diagnostic); the
// resource's own file is still written, since a human can add the missing entry by
// hand and the next placement for that directory re-detects the gap.
func (wb *writeBatch) appendKustomizationResource(
ctx context.Context,
event Event,
placement manifestanalyzer.PlacementResult,
) {
k := placement.Kustomization
entry := placement.Path
if dir := path.Dir(k.Path); dir != "." {
if rel, err := filepath.Rel(dir, placement.Path); err == nil {
entry = filepath.ToSlash(rel)
}
}
buf := wb.buffer(k.Path)
if buf.current == nil {
// The kustomization vanished within this batch; nothing to edit — and the file it
// would have registered is now outside every render, which is the same user-visible
// outcome as a failed edit, so it is counted as one.
recordKustomizationEntry(ctx, wb.target, kustomizationEntryFailed)
return
}
res, diags := manifestedit.AppendKustomizationResource(k.Path, buf.current, entry)
switch res.Mode {
case manifestedit.EditPatched:
buf.current = res.Content
recordKustomizationEntry(ctx, wb.target, kustomizationEntryAdded)
log.FromContext(ctx).Info("Added resources: entry for new file",
"kustomization", k.Path, "entry", entry, "resource", event.Identifier.String())
case manifestedit.EditNoChange:
recordKustomizationEntry(ctx, wb.target, kustomizationEntryNoChange)
case manifestedit.EditSkipped, manifestedit.EditDeleted, manifestedit.EditWholeReplace:
// The document is committed and its resources: entry is not, so kustomize will never
// build the file: it is in Git, it looks mirrored, and nothing applies it. The counter
// is the only signal that is not a log line.
recordKustomizationEntry(ctx, wb.target, kustomizationEntryFailed)
log.FromContext(ctx).Info("Could not add resources: entry for new file",
"kustomization", k.Path, "entry", entry, "resource", event.Identifier.String())
logManifestDiagnostics(ctx, diags)
}
}
// applyFieldPatch folds a subresource field-patch event into the batch: it locates the
// existing managed parent document by content identity and sets only the patch's
// declared field paths via manifestedit.PatchFields, preserving every other byte.
//
// Two deliberate refusals make this safe for a partial intent:
// - There is NO creation path. A patch whose parent is absent from Git is dropped,
// because fabricating the parent would mean guessing every unaudited field.
// - The renderer is NOT injected. A document that cannot be patched field-by-field
// is SKIPPED, not whole-replaced — a whole-replace from the partial desired would
// delete every field the subresource did not mention. An encrypted parent is
// likewise skipped (PatchFields inherits the SOPS refusal from Decide).
//
// The document index is re-derived from the buffer's CURRENT bytes so an earlier event
// in the same batch that shifted a multi-document file does not misdirect the edit.
func (wb *writeBatch) applyFieldPatch(ctx context.Context, event Event) error {
filePath, id, ok := wb.resolveFieldPatchTarget(event)
if !ok {
log.FromContext(ctx).Info("Dropping field patch: parent manifest not present in Git",
"resource", event.Identifier.String(), "source", event.FieldPatch.Source,
"reason", "subresource_patch_no_parent")
return nil
}
assignments := event.FieldPatch.Assignments
dm := wb.store.ByManifestIdentity[id]
governed := dm != nil && dm.Overrides != nil
if governed {
assignments = wb.routeGovernedFieldAssignments(ctx, event, dm, assignments)
// A routed scale changes only the kustomization entry, which still moves what this
// document renders to — so it must be declared, or the oracle would read its own
// intended write as collateral damage. It is UNCHECKED because a field patch carries
// a few audited assignments, never a whole object to compare the render against: the
// oracle can still prove the write disturbs nothing else, but not that it landed.
wb.putToKustomize = true
wb.intend(fieldPatchIntent(filePath, id, governed))
if len(assignments) == 0 {
return nil
}
}
buf := wb.buffer(filePath)
idx, found := currentDocIndex(filePath, buf.current, id)
if !found {
// An earlier event in this batch already removed the document; nothing to patch.
return nil
}
res, diags := manifestedit.PatchFields(
buf.current, idx, id, assignments, manifestedit.EditOptions{},
)
switch res.Mode {
case manifestedit.EditPatched:
buf.current = res.Content
if !governed {
wb.intend(fieldPatchIntent(filePath, id, false))
}
case manifestedit.EditNoChange, manifestedit.EditDeleted:
// No-op: the audited value already matched (or, impossible here, a delete).
case manifestedit.EditSkipped, manifestedit.EditWholeReplace:
// EditSkipped (encrypted, non-editable, or snapshot drift), or a defensive
// EditWholeReplace we must never apply from a partial desired.
log.FromContext(ctx).Info("Field patch not applied: parent is encrypted or not field-patchable",
"resource", event.Identifier.String(), "source", event.FieldPatch.Source,
"reason", "subresource_patch_unsafe")
logManifestDiagnostics(ctx, diags)
}
return nil
}
// routeGovernedFieldAssignments diverts a spec.replicas assignment whose value a
// replicas override governs to its kustomization entry (the /scale subresource
// case of the images/replicas edit-through) and returns the assignments the file
// patch should still apply. An
// ungoverned assignment — any other path, a non-integer value, no matching
// entry — keeps today's bounded file patch.
func (wb *writeBatch) routeGovernedFieldAssignments(
ctx context.Context,
event Event,
dm *manifestanalyzer.DocumentModel,
assignments []manifestedit.FieldAssignment,
) []manifestedit.FieldAssignment {
kept := make([]manifestedit.FieldAssignment, 0, len(assignments))
for _, a := range assignments {
if len(a.Path) == 2 && a.Path[0] == "spec" && a.Path[1] == "replicas" {
if count, isInt := assignmentInt64(a.Value); isInt {
if edit, governed := manifestanalyzer.ReplicaCountEdit(dm, count); governed {
wb.applyOverrideEdits(ctx, event, []manifestanalyzer.OverrideEdit{edit})
continue
}
}
}
kept = append(kept, a)
}
return kept
}
// assignmentInt64 reads a field-assignment value as a whole number (audit JSON
// may deliver it as int64 or float64).
func assignmentInt64(v any) (int64, bool) {
switch n := v.(type) {
case int64:
return n, true
case int:
return int64(n), true
case int32:
return int64(n), true
case float64:
if n == math.Trunc(n) {
return int64(n), true
}
}
return 0, false
}
// resolveFieldPatchTarget locates the parent manifest a field-patch event targets.
// The parent is resolved from its objectRef GVR through the same resource-identity
// inventory the GVR-only delete uses (PlanDelete), which the live-catalog mapper
// populates while scanning the GitTarget folder. The returned identity is the parent
// document's own manifest identity (full GVK from the committed YAML), so the patch
// is applied with the parent's real Kind, never one guessed from the subresource body.
//
// found is false when Git holds no managed document for the parent identity.
func (wb *writeBatch) resolveFieldPatchTarget(event Event) (string, manifestedit.Identity, bool) {
if action, emitted := manifestanalyzer.PlanDelete(wb.store, event.Identifier); emitted {
return action.Ref.FilePath, action.Identity, true
}
return "", manifestedit.Identity{}, false
}
// patchExisting edits the existing managed document for id in place via manifestedit,
// preserving the sibling documents' bytes and the target's hand-authored formatting.
// The no-op / patch / whole-replace / skip choice is a plan decision (Decide), not a
// per-event heuristic. The document position is re-derived from the buffer's CURRENT
// bytes (currentDocIndex), not the pre-batch store index, so an earlier event in the
// same batch that shifted a multi-document file does not misdirect this edit. A
// document the store located but an earlier event already removed is simply absent now,
// so there is nothing to patch.
//
// When the document is governed by a kustomize images/replicas override chain, the
// desired projection is first split: values the chain produces are restored to their
// source form (so the file keeps its bytes) and the divergence is routed to the
// override entries instead — see docs/design/support-boundary/finished/images-and-replicas-edit-through.md.
func (wb *writeBatch) patchExisting(
ctx context.Context,
event Event,
filePath string,
id manifestedit.Identity,
dm *manifestanalyzer.DocumentModel,
) (upsertOutcome, error) {
buf := wb.buffer(filePath)
idx, ok := currentDocIndex(filePath, buf.current, rawManifestIDForCurrentBytes(id, dm))
if !ok {
return upsertNoChange, nil
}
gitDoc, _ := manifestedit.NewDocumentAt(filePath, buf.current, idx)
desired := event.Object
if dm.NamespaceInheritedFromContext() && desired != nil {
desired = desired.DeepCopy()
desired.SetNamespace("")
}
projected, overrideEdits, err := projectThroughKustomize(
manifestreport.Project(desired), buf.current, idx, dm, wb.overlayAuthorKustomization(filePath))
if err != nil {
var fidelity *renderFidelityRefusedError
if errors.As(err, &fidelity) {
return upsertNoChange, renderFidelityRefusal(filePath, id, fidelity)
}
// The projection could not place the edit. Refusing the whole flush is the point: the
// alternative is to write the live object through and silently absorb the build's own
// output into the file that feeds it.
return upsertNoChange, sourceFormRefusal(filePath, id, err)
}
c := manifestedit.Comparison{
Git: gitDoc,
Desired: projected,
Options: manifestreport.EditOptions(),
}
res, diags := manifestedit.Apply(c, manifestedit.Decide(c))
outcome := upsertNoChange
switch res.Mode {
case manifestedit.EditPatched, manifestedit.EditWholeReplace:
buf.current = res.Content
outcome = upsertUpdated
case manifestedit.EditNoChange, manifestedit.EditSkipped, manifestedit.EditDeleted:
// No-op, an unsafe edit left untouched, or (impossible here) a delete: leave
// the bytes as they are. Surface a skip so an operator can see a document Git
// holds but the editor refused.
if res.Mode == manifestedit.EditSkipped {
logManifestDiagnostics(ctx, diags)
}
}
if wb.applyOverrideEdits(ctx, event, overrideEdits) {
outcome = upsertUpdated
}
// Declare what this document must render to. Attribution above decided WHERE the edit
// goes and is allowed to be wrong; the render precondition adjudicates it once the whole
// plan is known (see renderPrecondition).
//
// A GOVERNED document declares its intent even when its own bytes did not change, and
// that is not belt-and-braces — it is the difference between the oracle working and the
// oracle refusing perfectly good writes. An images: entry is shared: when two Deployments
// run the same image and are bumped together, the FIRST event's entry edit already moves
// what the second one renders to, so by the time the second is processed there is nothing
// left to write. Its render still moves, and it moves onto its own live state — that is
// the resource converging, not collateral damage, and only its declared intent says so.
//
// The oracle is armed for ANY document a render root produces, not only one an override
// chain governs, and the difference is a hole rather than a refinement. The source form
// leaves a field the build supplies to the source file — but where the live object and the
// render DISAGREE the user has changed something, and that change is written through. If a
// transformer or a patch owns that field it will be overridden right back, and the write
// never converges. Only the re-render can see that, and until now it did not run at all
// unless an images:/replicas: entry happened to exist somewhere in the chain.
if dm.Rendered != nil {
wb.putToKustomize = true
}
if outcome == upsertUpdated || dm.Overrides != nil {
wb.intend(intentFor(event.Object, filePath, dm.Overrides != nil))
}
return outcome, nil
}
// projectThroughKustomize turns the live projection into the SOURCE FORM of it: the object the
// file should hold once everything the build supplies is left to the build, plus the entry edits
// for the values an images:/replicas: entry supplies.
//
// A plain document uses its parsed Git object as its render. A kustomize document uses the
// DocumentModel's local render. In both cases, a rendered ${...} value that differs in live is
// refused before source-form projection can write the live expansion back into Git.
func projectThroughKustomize(
projected *unstructured.Unstructured,
content []byte,
idx int,
dm *manifestanalyzer.DocumentModel,
authorInto string,
) (*unstructured.Unstructured, []manifestanalyzer.OverrideEdit, error) {
gitRaw, parsed := gitDocRawObject(content, idx)
if !parsed {
return projected, nil, nil
}
rendered := gitRaw
if dm.Rendered != nil {
rendered = dm.Rendered.Object
}
if divergences := manifestanalyzer.RenderTokenDivergences(rendered, projected.Object); len(divergences) > 0 {
return nil, nil, &renderFidelityRefusedError{Divergences: divergences}
}
if dm.Rendered == nil {
return projected, nil, nil
}
return manifestanalyzer.SplitDesiredForOverrides(gitRaw, projected, dm.Rendered, authorInto)
}
// overlayAuthorKustomization is the kustomization the writer may author a NEW images:/replicas:
// entry into for an edit to filePath. It is set only when render-root scoping put filePath OUT of
// the write jail — a base document an overlay reads read-only — and the overlay has a supported
// render root of its own: then a value the base supplies can be overridden by authoring an entry
// in the overlay instead of refusing the base write. It is "" for a self-contained subtree and
// for an in-jail document, where the source file itself is writable.
func (wb *writeBatch) overlayAuthorKustomization(filePath string) string {
if wb.writeSubdir == "" || pathWithin(filePath, wb.writeSubdir) {
return ""
}
if k := wb.store.Kustomizations[wb.writeSubdir]; k != nil && !k.Unsupported {
return k.Path
}
return ""
}
// renderFidelityRefusedError travels from the projection seam to patchExisting, where the file
// and object identity are available to make a normal write-boundary refusal.
type renderFidelityRefusedError struct {
Divergences []manifestanalyzer.RenderDivergence
}
func (e *renderFidelityRefusedError) Error() string {
return "rendered token does not match live"
}
// sourceFormRefusal turns a projection that could not place an edit into the same reported
// refusal every other write-boundary violation surfaces as: GitPathAccepted=False / Stalled=True,
// naming the file and the object. It is not an internal error — the folder is fine and the
// operator is fine; the EDIT had nowhere honest to land, and saying so is the whole contract.
func sourceFormRefusal(filePath string, id manifestedit.Identity, err error) error {
return &manifestanalyzer.AcceptanceRefusedError{
Issues: []manifestanalyzer.AcceptanceIssue{{
Kind: manifestanalyzer.IssueUnplaceableEdit,
Path: filePath,
// Not solvable, and deliberately so: the alternative to refusing is aligning
// two lists by position, which is measurably wrong rather than merely risky
// (see the IssueUnplaceableEdit comment). Nobody can act on it.
Solvable: false,
Message: fmt.Sprintf("%s/%s in %s: %v",
id.Kind, id.Name, filePath, err),
}},
}
}
func renderFidelityRefusal(
filePath string,
id manifestedit.Identity,
fidelity *renderFidelityRefusedError,
) error {
issues := make([]manifestanalyzer.AcceptanceIssue, 0, len(fidelity.Divergences))
for _, divergence := range fidelity.Divergences {
issues = append(issues, manifestanalyzer.AcceptanceIssue{
Kind: manifestanalyzer.IssueRenderDoesNotMatchLive,
Path: filePath,
// A live value that diverges from what the folder renders is out-of-band
// substitution, not a render artifact, so whoever owns the deployment
// pipeline can reconcile the two.
Solvable: true,
Actor: manifestanalyzer.ActorPlatformOperator,
Field: divergence.Field,
Token: divergence.Token,
Message: fmt.Sprintf("%s/%s in %s: rendered token %q at %s does not match live",
id.Kind, id.Name, filePath, divergence.Token, divergence.Field),
})
}
return &manifestanalyzer.AcceptanceRefusedError{Issues: issues}
}
// renderPrecondition is the oracle, and it is a write-plan precondition like the three
// above it: it runs at the one moment the whole plan is known and before a single byte is
// touched, so a refusal aborts the flush and commits nothing.
//
// It only runs when the flush actually routed something through a kustomization. A repo
// with no override chain pays nothing, and a flush that changed no governed document has
// nothing for kustomize to adjudicate.
//
// A refusal is an AcceptanceRefusedError, which is the seam that carries it to the user as
// GitPathAccepted=False / Stalled=True with the file and object named. That is deliberate:
// render-attribution.md §7 is explicit that a proposal the renderer cannot vouch for
// "becomes a refused flush — that is the correct outcome and it must be reported, not
// absorbed." A resource we silently stop mirroring is the failure this path exists to
// prevent, so it must not be the failure this path introduces.
func (wb *writeBatch) renderPrecondition() error {
if !wb.putToKustomize {
return nil
}
before := make([]manifestedit.FileContent, 0, len(wb.contentByPath))
for _, path := range sortedContentKeys(wb.contentByPath) {
before = append(before, manifestedit.FileContent{Path: path, Content: wb.contentByPath[path]})
}
var refused *manifestanalyzer.RenderRefusedError
if err := manifestanalyzer.VerifyBatchRenders(before, wb.files(), wb.intents); err != nil {
if errors.As(err, &refused) {
issues := make([]manifestanalyzer.AcceptanceIssue, 0, len(refused.Reasons))
for _, reason := range refused.Reasons {
issues = append(issues, manifestanalyzer.AcceptanceIssue{
Kind: manifestanalyzer.IssueRenderRefused,
Message: reason,
// This refuses a WRITE, not a folder. Nobody can solve it from the
// repository or the GitTarget: the oracle refuses a write it cannot
// vouch for, and neither side can make it vouch.
Solvable: false,
})
}
return &manifestanalyzer.AcceptanceRefusedError{Issues: issues}
}
return err
}
return nil
}
// intend records what one document of this flush must render to, so the oracle can tell a
// change the flush MEANT from a change it merely caused. Everything not intended has to
// come out of the render untouched.
func (wb *writeBatch) intend(in manifestanalyzer.WriteIntent) {
if in.Kind == "" || in.Name == "" {
return // nothing addressable to check; the render comparison keys on kind+name
}
wb.intents = append(wb.intents, in)
}
// intentFor builds the intent for an ordinary object-bearing write: the document must