From 0776c4c1309e03a311c1bbe6c55e70186e725a1c Mon Sep 17 00:00:00 2001 From: Shivang Nagta Date: Tue, 23 Jun 2026 16:09:10 +0530 Subject: [PATCH 1/7] feat: add Spark History Server link to job details --- assets/databases/heimdall/tables/jobs.sql | 1 + internal/pkg/heimdall/job_dal.go | 4 +-- internal/pkg/heimdall/jobs_async.go | 4 +-- internal/pkg/heimdall/queries/job/select.sql | 3 ++- .../pkg/heimdall/queries/job/select_jobs.sql | 3 ++- .../heimdall/queries/job/status_update.sql | 3 ++- .../pkg/object/command/sparkeks/sparkeks.go | 4 +++ pkg/object/job/job.go | 27 ++++++++++--------- web/src/modules/Jobs/Helper.tsx | 1 + .../Jobs/JobDetails/JobInformationPane.tsx | 21 +++++++++++++++ 10 files changed, 51 insertions(+), 20 deletions(-) diff --git a/assets/databases/heimdall/tables/jobs.sql b/assets/databases/heimdall/tables/jobs.sql index 67aa6021..c7ff5fc9 100644 --- a/assets/databases/heimdall/tables/jobs.sql +++ b/assets/databases/heimdall/tables/jobs.sql @@ -21,6 +21,7 @@ create table if not exists jobs alter table jobs add column if not exists store_result_sync boolean not null default false; alter table jobs add column if not exists canceled_by varchar(64) null; +alter table jobs add column if not exists spark_application_id varchar(128) not null default ''; -- Originally had "cancelled_by" column and "cancelling" status, but we aren't british. Whoops. do $$ begin diff --git a/internal/pkg/heimdall/job_dal.go b/internal/pkg/heimdall/job_dal.go index 0fdc1e30..d333e900 100644 --- a/internal/pkg/heimdall/job_dal.go +++ b/internal/pkg/heimdall/job_dal.go @@ -210,7 +210,7 @@ func (h *Heimdall) getJob(ctx context.Context, j *jobRequest) (any, error) { var jobContext string if err := row.Scan(&r.SystemID, &r.Status, &r.Name, &r.Version, &r.Description, &jobContext, &r.Error, &r.User, &r.IsSync, - &r.CreatedAt, &r.UpdatedAt, &r.CommandID, &r.CommandName, &r.ClusterID, &r.ClusterName, &r.StoreResultSync, &r.CanceledBy); err != nil { + &r.CreatedAt, &r.UpdatedAt, &r.CommandID, &r.CommandName, &r.ClusterID, &r.ClusterName, &r.StoreResultSync, &r.CanceledBy, &r.SparkApplicationID); err != nil { if err == sql.ErrNoRows { return nil, ErrUnknownJobID } else { @@ -314,7 +314,7 @@ func (h *Heimdall) getJobs(ctx context.Context, f *database.Filter) (any, error) r := &job.Job{} if err := rows.Scan(&r.SystemID, &r.ID, &r.Status, &r.Name, &r.Version, &r.Description, &jobContext, &r.Error, &r.User, &r.IsSync, - &r.CreatedAt, &r.UpdatedAt, &r.CommandID, &r.CommandName, &r.ClusterID, &r.ClusterName, &r.StoreResultSync, &r.CanceledBy); err != nil { + &r.CreatedAt, &r.UpdatedAt, &r.CommandID, &r.CommandName, &r.ClusterID, &r.ClusterName, &r.StoreResultSync, &r.CanceledBy, &r.SparkApplicationID); err != nil { getJobsMethod.LogAndCountError(err, "scan") return nil, err } diff --git a/internal/pkg/heimdall/jobs_async.go b/internal/pkg/heimdall/jobs_async.go index 0f2e82ab..24417950 100644 --- a/internal/pkg/heimdall/jobs_async.go +++ b/internal/pkg/heimdall/jobs_async.go @@ -118,7 +118,7 @@ func (h *Heimdall) runAsyncJob(ctx context.Context, j *job.Job) error { } defer sess.Close() - if _, err := sess.Exec(queryJobStatusUpdate, status.Running, ``, j.SystemID); err != nil { + if _, err := sess.Exec(queryJobStatusUpdate, status.Running, ``, j.SparkApplicationID, j.SystemID); err != nil { return h.updateAsyncJobStatus(j, err) } @@ -154,7 +154,7 @@ func (h *Heimdall) updateAsyncJobStatus(j *job.Job, jobError error) error { } defer sess.Close() - if _, err := sess.Exec(queryJobStatusUpdate, j.Status, j.Error, j.SystemID); err != nil { + if _, err := sess.Exec(queryJobStatusUpdate, j.Status, j.Error, j.SparkApplicationID, j.SystemID); err != nil { // TODO: implement proper logging fmt.Println(`job status update error:`, err) } diff --git a/internal/pkg/heimdall/queries/job/select.sql b/internal/pkg/heimdall/queries/job/select.sql index 3019fcdd..5a2c2f82 100644 --- a/internal/pkg/heimdall/queries/job/select.sql +++ b/internal/pkg/heimdall/queries/job/select.sql @@ -15,7 +15,8 @@ select cl.cluster_id, cl.cluster_name, j.store_result_sync, - j.canceled_by + j.canceled_by, + j.spark_application_id from jobs j left join commands cm on cm.system_command_id = j.job_command_id diff --git a/internal/pkg/heimdall/queries/job/select_jobs.sql b/internal/pkg/heimdall/queries/job/select_jobs.sql index 74b26ac2..6e4b57c2 100644 --- a/internal/pkg/heimdall/queries/job/select_jobs.sql +++ b/internal/pkg/heimdall/queries/job/select_jobs.sql @@ -16,7 +16,8 @@ select cl.cluster_id, cl.cluster_name, j.store_result_sync, - j.canceled_by + j.canceled_by, + j.spark_application_id from jobs j join job_statuses js on js.job_status_id = j.job_status_id diff --git a/internal/pkg/heimdall/queries/job/status_update.sql b/internal/pkg/heimdall/queries/job/status_update.sql index 89786365..484163ce 100644 --- a/internal/pkg/heimdall/queries/job/status_update.sql +++ b/internal/pkg/heimdall/queries/job/status_update.sql @@ -2,6 +2,7 @@ update jobs set job_status_id = $1, job_error = left($2::text, 1024), + spark_application_id = coalesce(nullif($3::text, ''), spark_application_id), updated_at = extract(epoch from now())::int where - system_job_id = $3; + system_job_id = $4; diff --git a/internal/pkg/object/command/sparkeks/sparkeks.go b/internal/pkg/object/command/sparkeks/sparkeks.go index 4e44ab03..5c327364 100644 --- a/internal/pkg/object/command/sparkeks/sparkeks.go +++ b/internal/pkg/object/command/sparkeks/sparkeks.go @@ -929,6 +929,10 @@ func (e *executionContext) monitorJobAndCollectLogs(ctx context.Context) error { } finalSparkApp = sparkApp + // Capture Spark's runtime application id (used by the Spark History Server) + if id := sparkApp.Status.SparkApplicationID; id != "" { + e.job.SparkApplicationID = id + } state := sparkApp.Status.AppState.State if time.Since(lastReport) >= statusReportInterval || isTerminalState(state) { diff --git a/pkg/object/job/job.go b/pkg/object/job/job.go index 2ca17142..490024b0 100644 --- a/pkg/object/job/job.go +++ b/pkg/object/job/job.go @@ -10,19 +10,20 @@ import ( ) type Job struct { - object.Object `yaml:",inline" json:",inline"` - Status status.Status `yaml:"status,omitempty" json:"status,omitempty"` - IsSync bool `yaml:"is_sync,omitempty" json:"is_sync,omitempty"` - StoreResultSync bool `yaml:"store_result_sync,omitempty" json:"store_result_sync,omitempty"` - Error string `yaml:"error,omitempty" json:"error,omitempty"` - CommandCriteria *set.Set[string] `yaml:"command_criteria,omitempty" json:"command_criteria,omitempty"` - ClusterCriteria *set.Set[string] `yaml:"cluster_criteria,omitempty" json:"cluster_criteria,omitempty"` - CommandID string `yaml:"command_id,omitempty" json:"command_id,omitempty"` - CommandName string `yaml:"command_name,omitempty" json:"command_name,omitempty"` - ClusterID string `yaml:"cluster_id,omitempty" json:"cluster_id,omitempty"` - ClusterName string `yaml:"cluster_name,omitempty" json:"cluster_name,omitempty"` - CanceledBy string `yaml:"canceled_by,omitempty" json:"canceled_by,omitempty"` - Result *result.Result `yaml:"result,omitempty" json:"result,omitempty"` + object.Object `yaml:",inline" json:",inline"` + Status status.Status `yaml:"status,omitempty" json:"status,omitempty"` + IsSync bool `yaml:"is_sync,omitempty" json:"is_sync,omitempty"` + StoreResultSync bool `yaml:"store_result_sync,omitempty" json:"store_result_sync,omitempty"` + Error string `yaml:"error,omitempty" json:"error,omitempty"` + CommandCriteria *set.Set[string] `yaml:"command_criteria,omitempty" json:"command_criteria,omitempty"` + ClusterCriteria *set.Set[string] `yaml:"cluster_criteria,omitempty" json:"cluster_criteria,omitempty"` + CommandID string `yaml:"command_id,omitempty" json:"command_id,omitempty"` + CommandName string `yaml:"command_name,omitempty" json:"command_name,omitempty"` + ClusterID string `yaml:"cluster_id,omitempty" json:"cluster_id,omitempty"` + ClusterName string `yaml:"cluster_name,omitempty" json:"cluster_name,omitempty"` + CanceledBy string `yaml:"canceled_by,omitempty" json:"canceled_by,omitempty"` + SparkApplicationID string `yaml:"spark_application_id,omitempty" json:"spark_application_id,omitempty"` + Result *result.Result `yaml:"result,omitempty" json:"result,omitempty"` } func (j *Job) Init() error { diff --git a/web/src/modules/Jobs/Helper.tsx b/web/src/modules/Jobs/Helper.tsx index b1c5698a..b5f20bed 100644 --- a/web/src/modules/Jobs/Helper.tsx +++ b/web/src/modules/Jobs/Helper.tsx @@ -39,6 +39,7 @@ export type JobType = { cluster_id: string cluster_name: string canceled_by?: string + spark_application_id?: string error?: string context?: { properties: { diff --git a/web/src/modules/Jobs/JobDetails/JobInformationPane.tsx b/web/src/modules/Jobs/JobDetails/JobInformationPane.tsx index 55608078..1479e921 100644 --- a/web/src/modules/Jobs/JobDetails/JobInformationPane.tsx +++ b/web/src/modules/Jobs/JobDetails/JobInformationPane.tsx @@ -124,6 +124,27 @@ const JobInformationPane = ({ ]} isTwoColumns /> + {!!jobData?.spark_application_id && ( + + Spark History + + + ), + check: true, + }, + ]} + /> + )} )} From a022cc53c401a822adb458212981139885afdcf0 Mon Sep 17 00:00:00 2001 From: Shivang Nagta Date: Sun, 28 Jun 2026 18:09:32 +0530 Subject: [PATCH 2/7] Add a generic extra_job_attributes column in jobs --- assets/databases/heimdall/tables/jobs.sql | 2 +- internal/pkg/heimdall/job_dal.go | 28 ++++++++++-- internal/pkg/heimdall/jobs_async.go | 16 ++++++- internal/pkg/heimdall/queries/job/select.sql | 2 +- .../pkg/heimdall/queries/job/select_jobs.sql | 2 +- .../heimdall/queries/job/status_update.sql | 2 +- .../pkg/object/command/sparkeks/sparkeks.go | 14 ++++-- pkg/object/job/job.go | 38 ++++++++++------ web/src/modules/Jobs/Helper.tsx | 2 +- .../Jobs/JobDetails/JobInformationPane.tsx | 43 +++++++++++-------- 10 files changed, 104 insertions(+), 45 deletions(-) diff --git a/assets/databases/heimdall/tables/jobs.sql b/assets/databases/heimdall/tables/jobs.sql index c7ff5fc9..75d3a793 100644 --- a/assets/databases/heimdall/tables/jobs.sql +++ b/assets/databases/heimdall/tables/jobs.sql @@ -21,7 +21,7 @@ create table if not exists jobs alter table jobs add column if not exists store_result_sync boolean not null default false; alter table jobs add column if not exists canceled_by varchar(64) null; -alter table jobs add column if not exists spark_application_id varchar(128) not null default ''; +alter table jobs add column if not exists extra_job_attributes jsonb not null default '{}'::jsonb; -- Originally had "cancelled_by" column and "cancelling" status, but we aren't british. Whoops. do $$ begin diff --git a/internal/pkg/heimdall/job_dal.go b/internal/pkg/heimdall/job_dal.go index d333e900..3616c6de 100644 --- a/internal/pkg/heimdall/job_dal.go +++ b/internal/pkg/heimdall/job_dal.go @@ -207,10 +207,10 @@ func (h *Heimdall) getJob(ctx context.Context, j *jobRequest) (any, error) { }, } - var jobContext string + var jobContext, extraJobAttributes string if err := row.Scan(&r.SystemID, &r.Status, &r.Name, &r.Version, &r.Description, &jobContext, &r.Error, &r.User, &r.IsSync, - &r.CreatedAt, &r.UpdatedAt, &r.CommandID, &r.CommandName, &r.ClusterID, &r.ClusterName, &r.StoreResultSync, &r.CanceledBy, &r.SparkApplicationID); err != nil { + &r.CreatedAt, &r.UpdatedAt, &r.CommandID, &r.CommandName, &r.ClusterID, &r.ClusterName, &r.StoreResultSync, &r.CanceledBy, &extraJobAttributes); err != nil { if err == sql.ErrNoRows { return nil, ErrUnknownJobID } else { @@ -218,6 +218,11 @@ func (h *Heimdall) getJob(ctx context.Context, j *jobRequest) (any, error) { } } + if err := parseExtraJobAttributes(r, extraJobAttributes); err != nil { + getJobMethod.LogAndCountError(err, "parse_extra_job_attributes") + return nil, err + } + if err := jobParseContextAndTags(r, jobContext, sess); err != nil { getJobMethod.LogAndCountError(err, "job_parse_context_and_tags") return nil, err @@ -310,15 +315,20 @@ func (h *Heimdall) getJobs(ctx context.Context, f *database.Filter) (any, error) for rows.Next() { - jobContext := `` + jobContext, extraJobAttributes := ``, `` r := &job.Job{} if err := rows.Scan(&r.SystemID, &r.ID, &r.Status, &r.Name, &r.Version, &r.Description, &jobContext, &r.Error, &r.User, &r.IsSync, - &r.CreatedAt, &r.UpdatedAt, &r.CommandID, &r.CommandName, &r.ClusterID, &r.ClusterName, &r.StoreResultSync, &r.CanceledBy, &r.SparkApplicationID); err != nil { + &r.CreatedAt, &r.UpdatedAt, &r.CommandID, &r.CommandName, &r.ClusterID, &r.ClusterName, &r.StoreResultSync, &r.CanceledBy, &extraJobAttributes); err != nil { getJobsMethod.LogAndCountError(err, "scan") return nil, err } + if err := parseExtraJobAttributes(r, extraJobAttributes); err != nil { + getJobsMethod.LogAndCountError(err, "parse_extra_job_attributes") + return nil, err + } + if err := jobParseContextAndTags(r, jobContext, sess); err != nil { getJobsMethod.LogAndCountError(err, "job_parse_context_and_tags") return nil, err @@ -377,6 +387,16 @@ func (h *Heimdall) getJobStatus(ctx context.Context, j *jobRequest) (any, error) } +func parseExtraJobAttributes(j *job.Job, extraJobAttributes string) error { + if extraJobAttributes == `` || extraJobAttributes == `{}` { + return nil + } + if err := json.Unmarshal([]byte(extraJobAttributes), &j.ExtraJobAttributes); err != nil { + j.ExtraJobAttributes = nil + } + return nil +} + func jobParseContextAndTags(j *job.Job, jobContext string, sess *database.Session) (err error) { // ...and add job context diff --git a/internal/pkg/heimdall/jobs_async.go b/internal/pkg/heimdall/jobs_async.go index 24417950..303df299 100644 --- a/internal/pkg/heimdall/jobs_async.go +++ b/internal/pkg/heimdall/jobs_async.go @@ -3,6 +3,7 @@ package heimdall import ( "context" _ "embed" + "encoding/json" "fmt" "time" @@ -12,6 +13,17 @@ import ( "github.com/patterninc/heimdall/pkg/object/job/status" ) +func extraJobAttributesJSON(j *job.Job) string { + if len(j.ExtraJobAttributes) == 0 { + return `` + } + data, err := json.Marshal(j.ExtraJobAttributes) + if err != nil { + return `` + } + return string(data) +} + const ( formatErrUnknownCommand = "unknown command: %s" formatErrUnknownCluster = "unknown cluster: %s" @@ -118,7 +130,7 @@ func (h *Heimdall) runAsyncJob(ctx context.Context, j *job.Job) error { } defer sess.Close() - if _, err := sess.Exec(queryJobStatusUpdate, status.Running, ``, j.SparkApplicationID, j.SystemID); err != nil { + if _, err := sess.Exec(queryJobStatusUpdate, status.Running, ``, extraJobAttributesJSON(j), j.SystemID); err != nil { return h.updateAsyncJobStatus(j, err) } @@ -154,7 +166,7 @@ func (h *Heimdall) updateAsyncJobStatus(j *job.Job, jobError error) error { } defer sess.Close() - if _, err := sess.Exec(queryJobStatusUpdate, j.Status, j.Error, j.SparkApplicationID, j.SystemID); err != nil { + if _, err := sess.Exec(queryJobStatusUpdate, j.Status, j.Error, extraJobAttributesJSON(j), j.SystemID); err != nil { // TODO: implement proper logging fmt.Println(`job status update error:`, err) } diff --git a/internal/pkg/heimdall/queries/job/select.sql b/internal/pkg/heimdall/queries/job/select.sql index 5a2c2f82..2f76d625 100644 --- a/internal/pkg/heimdall/queries/job/select.sql +++ b/internal/pkg/heimdall/queries/job/select.sql @@ -16,7 +16,7 @@ select cl.cluster_name, j.store_result_sync, j.canceled_by, - j.spark_application_id + j.extra_job_attributes from jobs j left join commands cm on cm.system_command_id = j.job_command_id diff --git a/internal/pkg/heimdall/queries/job/select_jobs.sql b/internal/pkg/heimdall/queries/job/select_jobs.sql index 6e4b57c2..53490d4d 100644 --- a/internal/pkg/heimdall/queries/job/select_jobs.sql +++ b/internal/pkg/heimdall/queries/job/select_jobs.sql @@ -17,7 +17,7 @@ select cl.cluster_name, j.store_result_sync, j.canceled_by, - j.spark_application_id + j.extra_job_attributes from jobs j join job_statuses js on js.job_status_id = j.job_status_id diff --git a/internal/pkg/heimdall/queries/job/status_update.sql b/internal/pkg/heimdall/queries/job/status_update.sql index 484163ce..129e9911 100644 --- a/internal/pkg/heimdall/queries/job/status_update.sql +++ b/internal/pkg/heimdall/queries/job/status_update.sql @@ -2,7 +2,7 @@ update jobs set job_status_id = $1, job_error = left($2::text, 1024), - spark_application_id = coalesce(nullif($3::text, ''), spark_application_id), + extra_job_attributes = coalesce(nullif($3::text, '')::jsonb, extra_job_attributes), updated_at = extract(epoch from now())::int where system_job_id = $4; diff --git a/internal/pkg/object/command/sparkeks/sparkeks.go b/internal/pkg/object/command/sparkeks/sparkeks.go index 5c327364..bf76caa3 100644 --- a/internal/pkg/object/command/sparkeks/sparkeks.go +++ b/internal/pkg/object/command/sparkeks/sparkeks.go @@ -43,6 +43,7 @@ const ( statusReportInterval = 30 * time.Second defaultNamespace = "default" applicationPrefix = "spark-sql-job" + sparkHistoryAttribute = "Spark History" awsRegionEnvVar = "AWS_REGION" s3Prefix = "s3://" s3aPrefix = "s3a://" @@ -119,6 +120,7 @@ type clusterContext struct { KubernetesClusterName *string `yaml:"cluster_name,omitempty" json:"cluster_name,omitempty"` SparkApplicationFile string `yaml:"spark_application_file,omitempty" json:"spark_application_file,omitempty"` RequiredSparkSQLExtensions string `yaml:"required_spark_sql_extensions,omitempty" json:"required_spark_sql_extensions,omitempty"` + SparkHistoryURL string `yaml:"spark_history_url,omitempty" json:"spark_history_url,omitempty"` } // executionContext holds the final resolved configuration and clients for a job execution. @@ -929,9 +931,15 @@ func (e *executionContext) monitorJobAndCollectLogs(ctx context.Context) error { } finalSparkApp = sparkApp - // Capture Spark's runtime application id (used by the Spark History Server) - if id := sparkApp.Status.SparkApplicationID; id != "" { - e.job.SparkApplicationID = id + + if id := sparkApp.Status.SparkApplicationID; id != "" && e.clusterContext.SparkHistoryURL != "" { + if e.job.ExtraJobAttributes == nil { + e.job.ExtraJobAttributes = make(map[string]job.Attribute) + } + e.job.ExtraJobAttributes[sparkHistoryAttribute] = job.Attribute{ + Kind: job.AttributeKindLink, + Value: fmt.Sprintf("%s/history/%s/jobs/", e.clusterContext.SparkHistoryURL, id), + } } state := sparkApp.Status.AppState.State diff --git a/pkg/object/job/job.go b/pkg/object/job/job.go index 490024b0..47553c30 100644 --- a/pkg/object/job/job.go +++ b/pkg/object/job/job.go @@ -11,19 +11,31 @@ import ( type Job struct { object.Object `yaml:",inline" json:",inline"` - Status status.Status `yaml:"status,omitempty" json:"status,omitempty"` - IsSync bool `yaml:"is_sync,omitempty" json:"is_sync,omitempty"` - StoreResultSync bool `yaml:"store_result_sync,omitempty" json:"store_result_sync,omitempty"` - Error string `yaml:"error,omitempty" json:"error,omitempty"` - CommandCriteria *set.Set[string] `yaml:"command_criteria,omitempty" json:"command_criteria,omitempty"` - ClusterCriteria *set.Set[string] `yaml:"cluster_criteria,omitempty" json:"cluster_criteria,omitempty"` - CommandID string `yaml:"command_id,omitempty" json:"command_id,omitempty"` - CommandName string `yaml:"command_name,omitempty" json:"command_name,omitempty"` - ClusterID string `yaml:"cluster_id,omitempty" json:"cluster_id,omitempty"` - ClusterName string `yaml:"cluster_name,omitempty" json:"cluster_name,omitempty"` - CanceledBy string `yaml:"canceled_by,omitempty" json:"canceled_by,omitempty"` - SparkApplicationID string `yaml:"spark_application_id,omitempty" json:"spark_application_id,omitempty"` - Result *result.Result `yaml:"result,omitempty" json:"result,omitempty"` + Status status.Status `yaml:"status,omitempty" json:"status,omitempty"` + IsSync bool `yaml:"is_sync,omitempty" json:"is_sync,omitempty"` + StoreResultSync bool `yaml:"store_result_sync,omitempty" json:"store_result_sync,omitempty"` + Error string `yaml:"error,omitempty" json:"error,omitempty"` + CommandCriteria *set.Set[string] `yaml:"command_criteria,omitempty" json:"command_criteria,omitempty"` + ClusterCriteria *set.Set[string] `yaml:"cluster_criteria,omitempty" json:"cluster_criteria,omitempty"` + CommandID string `yaml:"command_id,omitempty" json:"command_id,omitempty"` + CommandName string `yaml:"command_name,omitempty" json:"command_name,omitempty"` + ClusterID string `yaml:"cluster_id,omitempty" json:"cluster_id,omitempty"` + ClusterName string `yaml:"cluster_name,omitempty" json:"cluster_name,omitempty"` + CanceledBy string `yaml:"canceled_by,omitempty" json:"canceled_by,omitempty"` + ExtraJobAttributes map[string]Attribute `yaml:"extra_job_attributes,omitempty" json:"extra_job_attributes,omitempty"` + Result *result.Result `yaml:"result,omitempty" json:"result,omitempty"` +} + +// Attribute kinds for ExtraJobAttributes. Kind tells the UI how to render Value. +const ( + AttributeKindLink = "link" + AttributeKindText = "text" +) + +// Attribute is a generic, plugin-agnostic value surfaced on a job in the UI. +type Attribute struct { + Kind string `yaml:"kind,omitempty" json:"kind,omitempty"` + Value string `yaml:"value,omitempty" json:"value,omitempty"` } func (j *Job) Init() error { diff --git a/web/src/modules/Jobs/Helper.tsx b/web/src/modules/Jobs/Helper.tsx index b5f20bed..8f45d02e 100644 --- a/web/src/modules/Jobs/Helper.tsx +++ b/web/src/modules/Jobs/Helper.tsx @@ -39,7 +39,7 @@ export type JobType = { cluster_id: string cluster_name: string canceled_by?: string - spark_application_id?: string + extra_job_attributes?: Record error?: string context?: { properties: { diff --git a/web/src/modules/Jobs/JobDetails/JobInformationPane.tsx b/web/src/modules/Jobs/JobDetails/JobInformationPane.tsx index 1479e921..0588a2b1 100644 --- a/web/src/modules/Jobs/JobDetails/JobInformationPane.tsx +++ b/web/src/modules/Jobs/JobDetails/JobInformationPane.tsx @@ -13,6 +13,7 @@ const JobInformationPane = ({ isLoading, }: JobDataTypesProps): React.JSX.Element => { const router = useRouter() + const extraJobAttributes = Object.entries(jobData?.extra_job_attributes ?? {}) const jobdetailsData = [ { label: 'Version', data: jobData?.version, check: !!jobData?.version }, { @@ -124,25 +125,31 @@ const JobInformationPane = ({ ]} isTwoColumns /> - {!!jobData?.spark_application_id && ( + {extraJobAttributes.length > 0 && ( - Spark History - - - ), - check: true, - }, - ]} + data={extraJobAttributes.map(([label, attr]) => + attr?.kind === 'link' + ? { + label: '', + data: ( + + ), + check: true, + } + : { label, data: attr?.value, check: true }, + )} /> )} From 2caf7cd5b66ec93f1c0dcdfa78f5ceae27cfe1aa Mon Sep 17 00:00:00 2001 From: Shivang Nagta Date: Tue, 30 Jun 2026 22:43:32 +0530 Subject: [PATCH 3/7] make extra_job_attributes config-driven via cluster attributes --- internal/pkg/heimdall/job.go | 53 +++++++++++++++++++ .../pkg/object/command/sparkeks/sparkeks.go | 13 ++--- pkg/object/cluster/cluster.go | 3 ++ pkg/object/job/job.go | 13 +++++ 4 files changed, 72 insertions(+), 10 deletions(-) diff --git a/internal/pkg/heimdall/job.go b/internal/pkg/heimdall/job.go index 1aefee49..71fcbd68 100644 --- a/internal/pkg/heimdall/job.go +++ b/internal/pkg/heimdall/job.go @@ -6,10 +6,12 @@ import ( _ "embed" "encoding/json" "fmt" + "io" "math/big" "net/http" "os" "strings" + "text/template" "time" "github.com/babourine/x/pkg/set" @@ -42,6 +44,7 @@ var ( ErrCallerNotAllowed = fmt.Errorf(`caller is not allowed to run this command`) runJobMethod = telemetry.NewMethod("runJob", "heimdall") cancelJobMethod = telemetry.NewMethod("db_connection", "cancel_job") + renderAttributesMethod = telemetry.NewMethod("renderAttributes", "heimdall") ) //go:embed queries/job/status_cancel_update.sql @@ -174,6 +177,8 @@ func (h *Heimdall) runJob(ctx context.Context, j *job.Job, command *command.Comm return nil } + renderJobAttributes(j, command, cluster) + // Handle plugin execution result (only if not canceled) if jobErr != nil { j.Status = jobStatus.Failed @@ -195,6 +200,54 @@ func (h *Heimdall) runJob(ctx context.Context, j *job.Job, command *command.Comm } +type attributeTemplateData struct { + Job *job.Job + Outputs map[string]string + Command map[string]any + Cluster map[string]any +} + +func renderJobAttributes(j *job.Job, command *command.Command, cluster *cluster.Cluster) { + + if len(cluster.Attributes) == 0 { + return + } + + data := attributeTemplateData{Job: j, Outputs: j.Outputs()} + if data.Outputs == nil { + data.Outputs = map[string]string{} + } + if command.Context != nil { + data.Command = map[string]any(*command.Context) + } + if cluster.Context != nil { + data.Cluster = map[string]any(*cluster.Context) + } + + for label, attr := range cluster.Attributes { + + parsed, err := template.New(`attribute`).Parse(attr.Value) + if err != nil { + renderAttributesMethod.LogAndCountError(fmt.Errorf("attribute %q: invalid template: %w", label, err)) + continue + } + + var value strings.Builder + if err := parsed.Option(`missingkey=error`).Execute(&value, data); err != nil { + if structuralErr := parsed.Option(`missingkey=zero`).Execute(io.Discard, data); structuralErr != nil { + renderAttributesMethod.LogAndCountError(fmt.Errorf("attribute %q: %w", label, structuralErr)) + } + continue + } + + if j.ExtraJobAttributes == nil { + j.ExtraJobAttributes = make(map[string]job.Attribute) + } + j.ExtraJobAttributes[label] = job.Attribute{Kind: attr.Kind, Value: value.String()} + } + +} + func (h *Heimdall) storeResults(runtime *plugin.Runtime, j *job.Job) error { // do we have result to be written? if j.Result == nil { diff --git a/internal/pkg/object/command/sparkeks/sparkeks.go b/internal/pkg/object/command/sparkeks/sparkeks.go index bf76caa3..e497ef50 100644 --- a/internal/pkg/object/command/sparkeks/sparkeks.go +++ b/internal/pkg/object/command/sparkeks/sparkeks.go @@ -43,7 +43,7 @@ const ( statusReportInterval = 30 * time.Second defaultNamespace = "default" applicationPrefix = "spark-sql-job" - sparkHistoryAttribute = "Spark History" + sparkApplicationIDOutput = "spark_application_id" awsRegionEnvVar = "AWS_REGION" s3Prefix = "s3://" s3aPrefix = "s3a://" @@ -120,7 +120,6 @@ type clusterContext struct { KubernetesClusterName *string `yaml:"cluster_name,omitempty" json:"cluster_name,omitempty"` SparkApplicationFile string `yaml:"spark_application_file,omitempty" json:"spark_application_file,omitempty"` RequiredSparkSQLExtensions string `yaml:"required_spark_sql_extensions,omitempty" json:"required_spark_sql_extensions,omitempty"` - SparkHistoryURL string `yaml:"spark_history_url,omitempty" json:"spark_history_url,omitempty"` } // executionContext holds the final resolved configuration and clients for a job execution. @@ -932,14 +931,8 @@ func (e *executionContext) monitorJobAndCollectLogs(ctx context.Context) error { finalSparkApp = sparkApp - if id := sparkApp.Status.SparkApplicationID; id != "" && e.clusterContext.SparkHistoryURL != "" { - if e.job.ExtraJobAttributes == nil { - e.job.ExtraJobAttributes = make(map[string]job.Attribute) - } - e.job.ExtraJobAttributes[sparkHistoryAttribute] = job.Attribute{ - Kind: job.AttributeKindLink, - Value: fmt.Sprintf("%s/history/%s/jobs/", e.clusterContext.SparkHistoryURL, id), - } + if id := sparkApp.Status.SparkApplicationID; id != "" { + e.job.SetOutput(sparkApplicationIDOutput, id) } state := sparkApp.Status.AppState.State diff --git a/pkg/object/cluster/cluster.go b/pkg/object/cluster/cluster.go index a31fc647..d6c3a7c0 100644 --- a/pkg/object/cluster/cluster.go +++ b/pkg/object/cluster/cluster.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/patterninc/heimdall/pkg/object" + "github.com/patterninc/heimdall/pkg/object/job" "github.com/patterninc/heimdall/pkg/object/status" "github.com/patterninc/heimdall/pkg/rbac" ) @@ -18,6 +19,8 @@ type Cluster struct { HealthCheck bool `yaml:"health_check,omitempty" json:"health_check,omitempty"` RBACNames []string `yaml:"rbacs,omitempty" json:"rbacs,omitempty"` RBACs []rbac.RBAC `yaml:"-" json:"-"` + + Attributes map[string]job.Attribute `yaml:"attributes,omitempty" json:"attributes,omitempty"` } type Clusters map[string]*Cluster diff --git a/pkg/object/job/job.go b/pkg/object/job/job.go index 47553c30..1407ffb1 100644 --- a/pkg/object/job/job.go +++ b/pkg/object/job/job.go @@ -24,6 +24,8 @@ type Job struct { CanceledBy string `yaml:"canceled_by,omitempty" json:"canceled_by,omitempty"` ExtraJobAttributes map[string]Attribute `yaml:"extra_job_attributes,omitempty" json:"extra_job_attributes,omitempty"` Result *result.Result `yaml:"result,omitempty" json:"result,omitempty"` + + outputs map[string]string } // Attribute kinds for ExtraJobAttributes. Kind tells the UI how to render Value. @@ -54,3 +56,14 @@ func (j *Job) Init() error { return nil } + +func (j *Job) SetOutput(key, value string) { + if j.outputs == nil { + j.outputs = make(map[string]string) + } + j.outputs[key] = value +} + +func (j *Job) Outputs() map[string]string { + return j.outputs +} From 9df3cab5e002d060c8bb04c5899d9eb6c0f9df71 Mon Sep 17 00:00:00 2001 From: Shivang Nagta Date: Wed, 1 Jul 2026 13:13:11 +0530 Subject: [PATCH 4/7] Refactor and Readme update --- README.md | 19 +++++++++++++++++++ configs/local.yaml | 11 ++++++++++- internal/pkg/heimdall/job.go | 14 +++++++------- internal/pkg/heimdall/jobs_async.go | 22 +++++++++++----------- pkg/object/cluster/cluster.go | 1 - pkg/object/job/job.go | 1 - 6 files changed, 47 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index d6b981b2..69ddf890 100644 --- a/README.md +++ b/README.md @@ -205,6 +205,25 @@ health_check: --- +## Config-Driven Job Attributes (For Static or Runtime generated metadata) + +Surface extra links or text on a job in the UI with minimal change in plugin code - by declaring `attributes` on a cluster. Each attribute is a template that Heimdall renders after the job runs and stores in the job's `extra_job_attributes`, which the UI then displays. Check local.yaml for the example. + +Each entry is `label → { kind, value }`: + +- **`kind`** tells the UI how to render `value`: `link` (clickable hyperlink) or `text`. +- **`value`** is a Go [`text/template`](https://pkg.go.dev/text/template) rendered over four namespaces: + - `.Job` — the job object (e.g. `.Job.ID`) + - `.Command` / `.Cluster` — the matched command's and cluster's `context` maps + - `.Outputs` — runtime values published by the plugin during execution + +Static attributes (built only from `.Job`, `.Command`, `.Cluster`) need no plugin changes. For runtime-discovered values, a plugin publishes to the outputs channel with a single call - for example, + +```go +job.SetOutput("some_runtime_metadata", value) +``` +--- + ## 👥 Credits **Heimdall** was created at **Pattern, Inc** by Stan Babourine, with contributions from Will Graham, Gaurav Warale and Josh Diaz. diff --git a/configs/local.yaml b/configs/local.yaml index e5042686..c6678b24 100644 --- a/configs/local.yaml +++ b/configs/local.yaml @@ -39,10 +39,19 @@ commands: # supported clusters clusters: - name: localhost-0.0.1 + attributes: + Test Link Label: + kind: link + value: "{{ .Cluster.test_url }}/complete_path" + Test Text Label: + kind: text + value: "{{ .Cluster.test_url }}/complete_path" status: active version: 0.0.1 description: Just a localhost health_check: true + context: + test_url: http://localhost:8080 tags: - type:localhost - - data:local \ No newline at end of file + - data:local diff --git a/internal/pkg/heimdall/job.go b/internal/pkg/heimdall/job.go index 71fcbd68..a78c265e 100644 --- a/internal/pkg/heimdall/job.go +++ b/internal/pkg/heimdall/job.go @@ -55,6 +55,13 @@ type commandOnCluster struct { cluster *cluster.Cluster } +type attributeTemplateData struct { + Job *job.Job + Outputs map[string]string + Command map[string]any + Cluster map[string]any +} + func (h *Heimdall) submitJob(ctx context.Context, j *job.Job) (any, error) { // set / add job properties @@ -200,13 +207,6 @@ func (h *Heimdall) runJob(ctx context.Context, j *job.Job, command *command.Comm } -type attributeTemplateData struct { - Job *job.Job - Outputs map[string]string - Command map[string]any - Cluster map[string]any -} - func renderJobAttributes(j *job.Job, command *command.Command, cluster *cluster.Cluster) { if len(cluster.Attributes) == 0 { diff --git a/internal/pkg/heimdall/jobs_async.go b/internal/pkg/heimdall/jobs_async.go index 303df299..0e317fa4 100644 --- a/internal/pkg/heimdall/jobs_async.go +++ b/internal/pkg/heimdall/jobs_async.go @@ -13,17 +13,6 @@ import ( "github.com/patterninc/heimdall/pkg/object/job/status" ) -func extraJobAttributesJSON(j *job.Job) string { - if len(j.ExtraJobAttributes) == 0 { - return `` - } - data, err := json.Marshal(j.ExtraJobAttributes) - if err != nil { - return `` - } - return string(data) -} - const ( formatErrUnknownCommand = "unknown command: %s" formatErrUnknownCluster = "unknown cluster: %s" @@ -47,6 +36,17 @@ var queryJobStatusUpdate string //go:embed queries/job/active_delete.sql var queryActiveJobDelete string +func extraJobAttributesJSON(j *job.Job) string { + if len(j.ExtraJobAttributes) == 0 { + return `` + } + data, err := json.Marshal(j.ExtraJobAttributes) + if err != nil { + return `` + } + return string(data) +} + func (h *Heimdall) getAsyncJobs(limit int) ([]*job.Job, error) { // Track DB connection for async jobs retrieval diff --git a/pkg/object/cluster/cluster.go b/pkg/object/cluster/cluster.go index d6c3a7c0..3e76b6d4 100644 --- a/pkg/object/cluster/cluster.go +++ b/pkg/object/cluster/cluster.go @@ -19,7 +19,6 @@ type Cluster struct { HealthCheck bool `yaml:"health_check,omitempty" json:"health_check,omitempty"` RBACNames []string `yaml:"rbacs,omitempty" json:"rbacs,omitempty"` RBACs []rbac.RBAC `yaml:"-" json:"-"` - Attributes map[string]job.Attribute `yaml:"attributes,omitempty" json:"attributes,omitempty"` } diff --git a/pkg/object/job/job.go b/pkg/object/job/job.go index 1407ffb1..d07aa7cc 100644 --- a/pkg/object/job/job.go +++ b/pkg/object/job/job.go @@ -24,7 +24,6 @@ type Job struct { CanceledBy string `yaml:"canceled_by,omitempty" json:"canceled_by,omitempty"` ExtraJobAttributes map[string]Attribute `yaml:"extra_job_attributes,omitempty" json:"extra_job_attributes,omitempty"` Result *result.Result `yaml:"result,omitempty" json:"result,omitempty"` - outputs map[string]string } From 3c3580638aa6d739dd03de3455a0798d678c46c8 Mon Sep 17 00:00:00 2001 From: Shivang Nagta Date: Wed, 1 Jul 2026 13:36:34 +0530 Subject: [PATCH 5/7] Improve attributes' value meaning in local.yaml --- configs/local.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/local.yaml b/configs/local.yaml index c6678b24..11c535df 100644 --- a/configs/local.yaml +++ b/configs/local.yaml @@ -42,10 +42,10 @@ clusters: attributes: Test Link Label: kind: link - value: "{{ .Cluster.test_url }}/complete_path" + value: "{{ .Cluster.test_url }}/remaining_static_or_runtime_generated_path" Test Text Label: kind: text - value: "{{ .Cluster.test_url }}/complete_path" + value: "static_or_runtime_generated_value_via_templating" status: active version: 0.0.1 description: Just a localhost From 584c8d1b865a5997929177aa0cb2da20d79f2440 Mon Sep 17 00:00:00 2001 From: Shivang Nagta Date: Wed, 1 Jul 2026 15:41:49 +0530 Subject: [PATCH 6/7] Add divider b/w hardcoded links and config generated links --- .../Jobs/JobDetails/JobInformationPane.tsx | 51 ++++++++++--------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/web/src/modules/Jobs/JobDetails/JobInformationPane.tsx b/web/src/modules/Jobs/JobDetails/JobInformationPane.tsx index 0588a2b1..7776bcf7 100644 --- a/web/src/modules/Jobs/JobDetails/JobInformationPane.tsx +++ b/web/src/modules/Jobs/JobDetails/JobInformationPane.tsx @@ -126,31 +126,32 @@ const JobInformationPane = ({ isTwoColumns /> {extraJobAttributes.length > 0 && ( - - attr?.kind === 'link' - ? { - label: '', - data: ( - - ), - check: true, - } - : { label, data: attr?.value, check: true }, - )} - /> + <> + + ({ + label: '', + data: + attr?.kind === 'link' ? ( + + ) : ( + + {label}: {attr?.value} + + ), + check: true, + }))} + isTwoColumns + /> + )} )} From 932a0465fd4af9ede21b92f2b32a097802926dc4 Mon Sep 17 00:00:00 2001 From: Shivang Nagta Date: Wed, 1 Jul 2026 16:24:10 +0530 Subject: [PATCH 7/7] rename extra_job_attributes to job_attributes --- README.md | 2 +- assets/databases/heimdall/tables/jobs.sql | 2 +- internal/pkg/heimdall/job.go | 6 ++--- internal/pkg/heimdall/job_dal.go | 24 +++++++++---------- internal/pkg/heimdall/jobs_async.go | 10 ++++---- internal/pkg/heimdall/queries/job/select.sql | 2 +- .../pkg/heimdall/queries/job/select_jobs.sql | 2 +- .../heimdall/queries/job/status_update.sql | 2 +- pkg/object/cluster/cluster.go | 10 ++++---- pkg/object/job/job.go | 6 ++--- web/src/modules/Jobs/Helper.tsx | 2 +- .../Jobs/JobDetails/JobInformationPane.tsx | 12 ++++++---- 12 files changed, 42 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 69ddf890..4ba0327f 100644 --- a/README.md +++ b/README.md @@ -207,7 +207,7 @@ health_check: ## Config-Driven Job Attributes (For Static or Runtime generated metadata) -Surface extra links or text on a job in the UI with minimal change in plugin code - by declaring `attributes` on a cluster. Each attribute is a template that Heimdall renders after the job runs and stores in the job's `extra_job_attributes`, which the UI then displays. Check local.yaml for the example. +Surface extra links or text on a job in the UI with minimal change in plugin code - by declaring `attributes` on a cluster. Each attribute is a template that Heimdall renders after the job runs and stores in the job's `job_attributes`, which the UI then displays. Check local.yaml for the example. Each entry is `label → { kind, value }`: diff --git a/assets/databases/heimdall/tables/jobs.sql b/assets/databases/heimdall/tables/jobs.sql index 75d3a793..e373a83e 100644 --- a/assets/databases/heimdall/tables/jobs.sql +++ b/assets/databases/heimdall/tables/jobs.sql @@ -21,7 +21,7 @@ create table if not exists jobs alter table jobs add column if not exists store_result_sync boolean not null default false; alter table jobs add column if not exists canceled_by varchar(64) null; -alter table jobs add column if not exists extra_job_attributes jsonb not null default '{}'::jsonb; +alter table jobs add column if not exists job_attributes jsonb not null default '{}'::jsonb; -- Originally had "cancelled_by" column and "cancelling" status, but we aren't british. Whoops. do $$ begin diff --git a/internal/pkg/heimdall/job.go b/internal/pkg/heimdall/job.go index a78c265e..4eca5916 100644 --- a/internal/pkg/heimdall/job.go +++ b/internal/pkg/heimdall/job.go @@ -240,10 +240,10 @@ func renderJobAttributes(j *job.Job, command *command.Command, cluster *cluster. continue } - if j.ExtraJobAttributes == nil { - j.ExtraJobAttributes = make(map[string]job.Attribute) + if j.JobAttributes == nil { + j.JobAttributes = make(map[string]job.Attribute) } - j.ExtraJobAttributes[label] = job.Attribute{Kind: attr.Kind, Value: value.String()} + j.JobAttributes[label] = job.Attribute{Kind: attr.Kind, Value: value.String()} } } diff --git a/internal/pkg/heimdall/job_dal.go b/internal/pkg/heimdall/job_dal.go index 3616c6de..3a37658f 100644 --- a/internal/pkg/heimdall/job_dal.go +++ b/internal/pkg/heimdall/job_dal.go @@ -207,10 +207,10 @@ func (h *Heimdall) getJob(ctx context.Context, j *jobRequest) (any, error) { }, } - var jobContext, extraJobAttributes string + var jobContext, jobAttributes string if err := row.Scan(&r.SystemID, &r.Status, &r.Name, &r.Version, &r.Description, &jobContext, &r.Error, &r.User, &r.IsSync, - &r.CreatedAt, &r.UpdatedAt, &r.CommandID, &r.CommandName, &r.ClusterID, &r.ClusterName, &r.StoreResultSync, &r.CanceledBy, &extraJobAttributes); err != nil { + &r.CreatedAt, &r.UpdatedAt, &r.CommandID, &r.CommandName, &r.ClusterID, &r.ClusterName, &r.StoreResultSync, &r.CanceledBy, &jobAttributes); err != nil { if err == sql.ErrNoRows { return nil, ErrUnknownJobID } else { @@ -218,8 +218,8 @@ func (h *Heimdall) getJob(ctx context.Context, j *jobRequest) (any, error) { } } - if err := parseExtraJobAttributes(r, extraJobAttributes); err != nil { - getJobMethod.LogAndCountError(err, "parse_extra_job_attributes") + if err := parseJobAttributes(r, jobAttributes); err != nil { + getJobMethod.LogAndCountError(err, "parse_job_attributes") return nil, err } @@ -315,17 +315,17 @@ func (h *Heimdall) getJobs(ctx context.Context, f *database.Filter) (any, error) for rows.Next() { - jobContext, extraJobAttributes := ``, `` + jobContext, jobAttributes := ``, `` r := &job.Job{} if err := rows.Scan(&r.SystemID, &r.ID, &r.Status, &r.Name, &r.Version, &r.Description, &jobContext, &r.Error, &r.User, &r.IsSync, - &r.CreatedAt, &r.UpdatedAt, &r.CommandID, &r.CommandName, &r.ClusterID, &r.ClusterName, &r.StoreResultSync, &r.CanceledBy, &extraJobAttributes); err != nil { + &r.CreatedAt, &r.UpdatedAt, &r.CommandID, &r.CommandName, &r.ClusterID, &r.ClusterName, &r.StoreResultSync, &r.CanceledBy, &jobAttributes); err != nil { getJobsMethod.LogAndCountError(err, "scan") return nil, err } - if err := parseExtraJobAttributes(r, extraJobAttributes); err != nil { - getJobsMethod.LogAndCountError(err, "parse_extra_job_attributes") + if err := parseJobAttributes(r, jobAttributes); err != nil { + getJobsMethod.LogAndCountError(err, "parse_job_attributes") return nil, err } @@ -387,12 +387,12 @@ func (h *Heimdall) getJobStatus(ctx context.Context, j *jobRequest) (any, error) } -func parseExtraJobAttributes(j *job.Job, extraJobAttributes string) error { - if extraJobAttributes == `` || extraJobAttributes == `{}` { +func parseJobAttributes(j *job.Job, jobAttributes string) error { + if jobAttributes == `` || jobAttributes == `{}` { return nil } - if err := json.Unmarshal([]byte(extraJobAttributes), &j.ExtraJobAttributes); err != nil { - j.ExtraJobAttributes = nil + if err := json.Unmarshal([]byte(jobAttributes), &j.JobAttributes); err != nil { + j.JobAttributes = nil } return nil } diff --git a/internal/pkg/heimdall/jobs_async.go b/internal/pkg/heimdall/jobs_async.go index 0e317fa4..750ac9b3 100644 --- a/internal/pkg/heimdall/jobs_async.go +++ b/internal/pkg/heimdall/jobs_async.go @@ -36,11 +36,11 @@ var queryJobStatusUpdate string //go:embed queries/job/active_delete.sql var queryActiveJobDelete string -func extraJobAttributesJSON(j *job.Job) string { - if len(j.ExtraJobAttributes) == 0 { +func jobAttributesJSON(j *job.Job) string { + if len(j.JobAttributes) == 0 { return `` } - data, err := json.Marshal(j.ExtraJobAttributes) + data, err := json.Marshal(j.JobAttributes) if err != nil { return `` } @@ -130,7 +130,7 @@ func (h *Heimdall) runAsyncJob(ctx context.Context, j *job.Job) error { } defer sess.Close() - if _, err := sess.Exec(queryJobStatusUpdate, status.Running, ``, extraJobAttributesJSON(j), j.SystemID); err != nil { + if _, err := sess.Exec(queryJobStatusUpdate, status.Running, ``, jobAttributesJSON(j), j.SystemID); err != nil { return h.updateAsyncJobStatus(j, err) } @@ -166,7 +166,7 @@ func (h *Heimdall) updateAsyncJobStatus(j *job.Job, jobError error) error { } defer sess.Close() - if _, err := sess.Exec(queryJobStatusUpdate, j.Status, j.Error, extraJobAttributesJSON(j), j.SystemID); err != nil { + if _, err := sess.Exec(queryJobStatusUpdate, j.Status, j.Error, jobAttributesJSON(j), j.SystemID); err != nil { // TODO: implement proper logging fmt.Println(`job status update error:`, err) } diff --git a/internal/pkg/heimdall/queries/job/select.sql b/internal/pkg/heimdall/queries/job/select.sql index 2f76d625..fc0643da 100644 --- a/internal/pkg/heimdall/queries/job/select.sql +++ b/internal/pkg/heimdall/queries/job/select.sql @@ -16,7 +16,7 @@ select cl.cluster_name, j.store_result_sync, j.canceled_by, - j.extra_job_attributes + j.job_attributes from jobs j left join commands cm on cm.system_command_id = j.job_command_id diff --git a/internal/pkg/heimdall/queries/job/select_jobs.sql b/internal/pkg/heimdall/queries/job/select_jobs.sql index 53490d4d..2a2c9e99 100644 --- a/internal/pkg/heimdall/queries/job/select_jobs.sql +++ b/internal/pkg/heimdall/queries/job/select_jobs.sql @@ -17,7 +17,7 @@ select cl.cluster_name, j.store_result_sync, j.canceled_by, - j.extra_job_attributes + j.job_attributes from jobs j join job_statuses js on js.job_status_id = j.job_status_id diff --git a/internal/pkg/heimdall/queries/job/status_update.sql b/internal/pkg/heimdall/queries/job/status_update.sql index 129e9911..d044abaa 100644 --- a/internal/pkg/heimdall/queries/job/status_update.sql +++ b/internal/pkg/heimdall/queries/job/status_update.sql @@ -2,7 +2,7 @@ update jobs set job_status_id = $1, job_error = left($2::text, 1024), - extra_job_attributes = coalesce(nullif($3::text, '')::jsonb, extra_job_attributes), + job_attributes = coalesce(nullif($3::text, '')::jsonb, job_attributes), updated_at = extract(epoch from now())::int where system_job_id = $4; diff --git a/pkg/object/cluster/cluster.go b/pkg/object/cluster/cluster.go index 3e76b6d4..43009aec 100644 --- a/pkg/object/cluster/cluster.go +++ b/pkg/object/cluster/cluster.go @@ -15,11 +15,11 @@ var ( type Cluster struct { object.Object `yaml:",inline" json:",inline"` - Status status.Status `yaml:"status,omitempty" json:"status,omitempty"` - HealthCheck bool `yaml:"health_check,omitempty" json:"health_check,omitempty"` - RBACNames []string `yaml:"rbacs,omitempty" json:"rbacs,omitempty"` - RBACs []rbac.RBAC `yaml:"-" json:"-"` - Attributes map[string]job.Attribute `yaml:"attributes,omitempty" json:"attributes,omitempty"` + Status status.Status `yaml:"status,omitempty" json:"status,omitempty"` + HealthCheck bool `yaml:"health_check,omitempty" json:"health_check,omitempty"` + RBACNames []string `yaml:"rbacs,omitempty" json:"rbacs,omitempty"` + RBACs []rbac.RBAC `yaml:"-" json:"-"` + Attributes map[string]job.Attribute `yaml:"attributes,omitempty" json:"attributes,omitempty"` } type Clusters map[string]*Cluster diff --git a/pkg/object/job/job.go b/pkg/object/job/job.go index d07aa7cc..59185836 100644 --- a/pkg/object/job/job.go +++ b/pkg/object/job/job.go @@ -22,12 +22,12 @@ type Job struct { ClusterID string `yaml:"cluster_id,omitempty" json:"cluster_id,omitempty"` ClusterName string `yaml:"cluster_name,omitempty" json:"cluster_name,omitempty"` CanceledBy string `yaml:"canceled_by,omitempty" json:"canceled_by,omitempty"` - ExtraJobAttributes map[string]Attribute `yaml:"extra_job_attributes,omitempty" json:"extra_job_attributes,omitempty"` + JobAttributes map[string]Attribute `yaml:"job_attributes,omitempty" json:"job_attributes,omitempty"` Result *result.Result `yaml:"result,omitempty" json:"result,omitempty"` - outputs map[string]string + outputs map[string]string } -// Attribute kinds for ExtraJobAttributes. Kind tells the UI how to render Value. +// Attribute kinds for JobAttributes. Kind tells the UI how to render Value. const ( AttributeKindLink = "link" AttributeKindText = "text" diff --git a/web/src/modules/Jobs/Helper.tsx b/web/src/modules/Jobs/Helper.tsx index 8f45d02e..67d71c78 100644 --- a/web/src/modules/Jobs/Helper.tsx +++ b/web/src/modules/Jobs/Helper.tsx @@ -39,7 +39,7 @@ export type JobType = { cluster_id: string cluster_name: string canceled_by?: string - extra_job_attributes?: Record + job_attributes?: Record error?: string context?: { properties: { diff --git a/web/src/modules/Jobs/JobDetails/JobInformationPane.tsx b/web/src/modules/Jobs/JobDetails/JobInformationPane.tsx index 7776bcf7..59e5fe98 100644 --- a/web/src/modules/Jobs/JobDetails/JobInformationPane.tsx +++ b/web/src/modules/Jobs/JobDetails/JobInformationPane.tsx @@ -13,7 +13,7 @@ const JobInformationPane = ({ isLoading, }: JobDataTypesProps): React.JSX.Element => { const router = useRouter() - const extraJobAttributes = Object.entries(jobData?.extra_job_attributes ?? {}) + const jobAttributes = Object.entries(jobData?.job_attributes ?? {}) const jobdetailsData = [ { label: 'Version', data: jobData?.version, check: !!jobData?.version }, { @@ -125,11 +125,11 @@ const JobInformationPane = ({ ]} isTwoColumns /> - {extraJobAttributes.length > 0 && ( + {jobAttributes.length > 0 && ( <> ({ + data={jobAttributes.map(([label, attr]) => ({ label: '', data: attr?.kind === 'link' ? ( @@ -140,7 +140,11 @@ const JobInformationPane = ({ className='gap-1' > {label} - + ) : (