feat: add remote URL template loading and source tracking

README.md

@@ -195,7 +195,7 @@ Users must be in at least one role mapping (`org_role_mapping` or `user_role_map
 
 ### Groups and Templates
 
-Groups define pools of runners identified by labels. Each group can have multiple workflow dispatch templates defined inline or loaded from a separate file:
+Groups define pools of runners identified by labels. Each group can have multiple workflow dispatch templates defined inline, loaded from local files, or fetched from remote URLs:
 
 ```yaml
 groups:
@@ -218,10 +218,13 @@ groups:
             el-client: "geth"
             cl-client: "prysm"
             config: '{"network": "mainnet"}'
-      # Option 2: Load templates from files (paths relative to config file)
+      # Option 2: Load templates from local files (paths relative to config file)
       # workflow_dispatch_templates_files:
       #   - templates/hoodi.yaml
       #   - templates/mainnet.yaml
+      # Option 3: Load templates from remote URLs
+      # workflow_dispatch_templates_urls:
+      #   - https://raw.githubusercontent.com/myorg/templates/main/sync-tests.yaml
 ```
 
 Template file format (`templates/sync-tests.yaml`):
@@ -247,7 +250,7 @@ Template file format (`templates/sync-tests.yaml`):
     cl-client: "lighthouse"
 ```
 
-Both inline templates and file templates can be used together - file templates are appended to inline templates.
+All template sources can be used together - file and URL templates are appended to inline templates. The UI displays badges indicating the source of each template (inline, local file, or URL).
 
 ### Workflow Best Practices
 

config.example.yaml

@@ -70,11 +70,13 @@ groups:
         - self-hosted
         - synctest
         - Disk2TB
-      # Templates can be defined inline or loaded from separate files:
+      # Templates can be defined inline, loaded from local files, or fetched from remote URLs:
       # workflow_dispatch_templates_files:
       #   - templates/hoodi.yaml
       #   - templates/mainnet.yaml
-      # Both can be used together - file templates are appended to inline templates.
+      # workflow_dispatch_templates_urls:
+      #   - https://raw.githubusercontent.com/myorg/templates/main/sync-tests.yaml
+      # All sources can be used together - file and URL templates are appended to inline templates.
       workflow_dispatch_templates:
         - id: sync-test-hoodi-geth-prysm
           name: Sync Test (Hoodi) - geth/prysm

pkg/api/api.go

@@ -1992,6 +1992,8 @@ func SyncGroupsFromConfig(ctx context.Context, log logrus.FieldLogger, st store.
 				DefaultInputs: tmplCfg.Inputs,
 				Labels:        tmplCfg.Labels,
 				InConfig:      true,
+				SourceType:    tmplCfg.SourceType,
+				SourcePath:    tmplCfg.SourcePath,
 				CreatedAt:     now,
 				UpdatedAt:     now,
 			}

pkg/config/config.go

@@ -2,6 +2,8 @@ package config
 
 import (
 	"fmt"
+	"io"
+	"net/http"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -113,6 +115,7 @@ type Group struct {
 	RunnerLabels                   []string                   `yaml:"runner_labels"`
 	WorkflowDispatchTemplates      []WorkflowDispatchTemplate `yaml:"workflow_dispatch_templates"`
 	WorkflowDispatchTemplatesFiles []string                   `yaml:"workflow_dispatch_templates_files"`
+	WorkflowDispatchTemplatesURLs  []string                   `yaml:"workflow_dispatch_templates_urls"`
 }
 
 // WorkflowDispatchTemplate represents a workflow dispatch template configuration.
@@ -125,6 +128,8 @@ type WorkflowDispatchTemplate struct {
 	Ref        string            `yaml:"ref"`
 	Inputs     map[string]string `yaml:"inputs"`
 	Labels     map[string]string `yaml:"labels"`
+	SourceType string            `yaml:"-"` // "inline", "file", or "url" - set during loading
+	SourcePath string            `yaml:"-"` // filename or URL (empty for inline) - set during loading
 }
 
 // Load reads and parses configuration from a YAML file.
@@ -142,12 +147,20 @@ func Load(path string) (*Config, error) {
 		return nil, fmt.Errorf("parsing config file: %w", err)
 	}
 
+	// Mark inline templates with source type.
+	markInlineTemplates(&cfg)
+
 	// Load templates from external files.
 	configDir := filepath.Dir(path)
 	if err := loadTemplateFiles(&cfg, configDir); err != nil {
 		return nil, fmt.Errorf("loading template files: %w", err)
 	}
 
+	// Load templates from remote URLs.
+	if err := loadTemplateURLs(&cfg); err != nil {
+		return nil, fmt.Errorf("loading template URLs: %w", err)
+	}
+
 	// Apply defaults.
 	applyDefaults(&cfg)
 
@@ -159,6 +172,16 @@ func Load(path string) (*Config, error) {
 	return &cfg, nil
 }
 
+// markInlineTemplates marks templates defined inline in the config with source type.
+func markInlineTemplates(cfg *Config) {
+	for i := range cfg.Groups.GitHub {
+		for j := range cfg.Groups.GitHub[i].WorkflowDispatchTemplates {
+			cfg.Groups.GitHub[i].WorkflowDispatchTemplates[j].SourceType = "inline"
+			cfg.Groups.GitHub[i].WorkflowDispatchTemplates[j].SourcePath = ""
+		}
+	}
+}
+
 // loadTemplateFiles loads workflow dispatch templates from external files.
 func loadTemplateFiles(cfg *Config, configDir string) error {
 	for i := range cfg.Groups.GitHub {
@@ -187,6 +210,12 @@ func loadTemplateFiles(cfg *Config, configDir string) error {
 					templateFile, group.ID, err)
 			}
 
+			// Set source type and path for each template from file.
+			for j := range templates {
+				templates[j].SourceType = "file"
+				templates[j].SourcePath = templateFile
+			}
+
 			// Append templates from file to any inline templates.
 			group.WorkflowDispatchTemplates = append(group.WorkflowDispatchTemplates, templates...)
 		}
@@ -195,6 +224,63 @@ func loadTemplateFiles(cfg *Config, configDir string) error {
 	return nil
 }
 
+// loadTemplateURLs loads workflow dispatch templates from remote URLs.
+func loadTemplateURLs(cfg *Config) error {
+	client := &http.Client{
+		Timeout: 30 * time.Second,
+	}
+
+	for i := range cfg.Groups.GitHub {
+		group := &cfg.Groups.GitHub[i]
+
+		for _, templateURL := range group.WorkflowDispatchTemplatesURLs {
+			resp, err := client.Get(templateURL)
+			if err != nil {
+				return fmt.Errorf("fetching template URL %s for group %s: %w",
+					templateURL, group.ID, err)
+			}
+
+			data, err := io.ReadAll(resp.Body)
+
+			closeErr := resp.Body.Close()
+			if err != nil {
+				return fmt.Errorf("reading response from %s for group %s: %w",
+					templateURL, group.ID, err)
+			}
+
+			if closeErr != nil {
+				return fmt.Errorf("closing response body from %s for group %s: %w",
+					templateURL, group.ID, closeErr)
+			}
+
+			if resp.StatusCode != http.StatusOK {
+				return fmt.Errorf("fetching template URL %s for group %s: HTTP %d",
+					templateURL, group.ID, resp.StatusCode)
+			}
+
+			// Expand environment variables.
+			expanded := expandEnvVars(string(data))
+
+			var templates []WorkflowDispatchTemplate
+			if err := yaml.Unmarshal([]byte(expanded), &templates); err != nil {
+				return fmt.Errorf("parsing template URL %s for group %s: %w",
+					templateURL, group.ID, err)
+			}
+
+			// Set source type and path for each template from URL.
+			for j := range templates {
+				templates[j].SourceType = "url"
+				templates[j].SourcePath = templateURL
+			}
+
+			// Append templates from URL to any existing templates.
+			group.WorkflowDispatchTemplates = append(group.WorkflowDispatchTemplates, templates...)
+		}
+	}
+
+	return nil
+}
+
 // expandEnvVars replaces ${VAR} and $VAR patterns with environment variable values.
 func expandEnvVars(s string) string {
 	// Match ${VAR} pattern.

pkg/store/postgres.go

@@ -264,6 +264,17 @@ func (s *PostgresStore) Migrate(ctx context.Context) error {
 			created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP
 		)`,
 		`CREATE INDEX IF NOT EXISTS idx_auth_codes_expires ON auth_codes(expires_at)`,
+		// Migration: Add source_type and source_path columns to job_templates table.
+		`DO $$ BEGIN
+			ALTER TABLE job_templates ADD COLUMN source_type TEXT NOT NULL DEFAULT 'inline';
+		EXCEPTION
+			WHEN duplicate_column THEN NULL;
+		END $$`,
+		`DO $$ BEGIN
+			ALTER TABLE job_templates ADD COLUMN source_path TEXT NOT NULL DEFAULT '';
+		EXCEPTION
+			WHEN duplicate_column THEN NULL;
+		END $$`,
 	}
 
 	for _, migration := range migrations {
@@ -408,10 +419,11 @@ func (s *PostgresStore) CreateJobTemplate(ctx context.Context, template *JobTemp
 	}
 
 	_, err = s.db.ExecContext(ctx, `
-		INSERT INTO job_templates (id, group_id, name, owner, repo, workflow_id, ref, default_inputs, labels, in_config, created_at, updated_at)
-		VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
+		INSERT INTO job_templates (id, group_id, name, owner, repo, workflow_id, ref, default_inputs, labels, in_config, source_type, source_path, created_at, updated_at)
+		VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)
 	`, template.ID, template.GroupID, template.Name, template.Owner, template.Repo,
-		template.WorkflowID, template.Ref, string(inputsJSON), string(labelsJSON), template.InConfig, template.CreatedAt, template.UpdatedAt)
+		template.WorkflowID, template.Ref, string(inputsJSON), string(labelsJSON), template.InConfig,
+		template.SourceType, template.SourcePath, template.CreatedAt, template.UpdatedAt)
 
 	if err != nil {
 		return fmt.Errorf("inserting job_template: %w", err)
@@ -427,11 +439,11 @@ func (s *PostgresStore) GetJobTemplate(ctx context.Context, id string) (*JobTemp
 	var inputsJSON, labelsJSON sql.NullString
 
 	err := s.db.QueryRowContext(ctx, `
-		SELECT id, group_id, name, owner, repo, workflow_id, ref, default_inputs, labels, in_config, created_at, updated_at
+		SELECT id, group_id, name, owner, repo, workflow_id, ref, default_inputs, labels, in_config, source_type, source_path, created_at, updated_at
 		FROM job_templates WHERE id = $1
 	`, id).Scan(&template.ID, &template.GroupID, &template.Name, &template.Owner,
 		&template.Repo, &template.WorkflowID, &template.Ref, &inputsJSON, &labelsJSON,
-		&template.InConfig, &template.CreatedAt, &template.UpdatedAt)
+		&template.InConfig, &template.SourceType, &template.SourcePath, &template.CreatedAt, &template.UpdatedAt)
 
 	if err == sql.ErrNoRows {
 		return nil, nil
@@ -459,7 +471,7 @@ func (s *PostgresStore) GetJobTemplate(ctx context.Context, id string) (*JobTemp
 // ListJobTemplatesByGroup retrieves all job templates for a group.
 func (s *PostgresStore) ListJobTemplatesByGroup(ctx context.Context, groupID string) ([]*JobTemplate, error) {
 	rows, err := s.db.QueryContext(ctx, `
-		SELECT id, group_id, name, owner, repo, workflow_id, ref, default_inputs, labels, in_config, created_at, updated_at
+		SELECT id, group_id, name, owner, repo, workflow_id, ref, default_inputs, labels, in_config, source_type, source_path, created_at, updated_at
 		FROM job_templates WHERE group_id = $1 ORDER BY name
 	`, groupID)
 	if err != nil {
@@ -477,7 +489,7 @@ func (s *PostgresStore) ListJobTemplatesByGroup(ctx context.Context, groupID str
 
 		if err := rows.Scan(&template.ID, &template.GroupID, &template.Name, &template.Owner,
 			&template.Repo, &template.WorkflowID, &template.Ref, &inputsJSON, &labelsJSON,
-			&template.InConfig, &template.CreatedAt, &template.UpdatedAt); err != nil {
+			&template.InConfig, &template.SourceType, &template.SourcePath, &template.CreatedAt, &template.UpdatedAt); err != nil {
 			return nil, fmt.Errorf("scanning job_template: %w", err)
 		}
 
@@ -514,10 +526,10 @@ func (s *PostgresStore) UpdateJobTemplate(ctx context.Context, template *JobTemp
 	template.UpdatedAt = time.Now()
 
 	_, err = s.db.ExecContext(ctx, `
-		UPDATE job_templates SET name = $1, owner = $2, repo = $3, workflow_id = $4, ref = $5, default_inputs = $6, labels = $7, in_config = $8, updated_at = $9
-		WHERE id = $10
+		UPDATE job_templates SET name = $1, owner = $2, repo = $3, workflow_id = $4, ref = $5, default_inputs = $6, labels = $7, in_config = $8, source_type = $9, source_path = $10, updated_at = $11
+		WHERE id = $12
 	`, template.Name, template.Owner, template.Repo, template.WorkflowID, template.Ref,
-		string(inputsJSON), string(labelsJSON), template.InConfig, template.UpdatedAt, template.ID)
+		string(inputsJSON), string(labelsJSON), template.InConfig, template.SourceType, template.SourcePath, template.UpdatedAt, template.ID)
 
 	if err != nil {
 		return fmt.Errorf("updating job_template: %w", err)

pkg/store/sqlite.go

@@ -201,6 +201,9 @@ func (s *SQLiteStore) Migrate(ctx context.Context) error {
 			created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
 		)`,
 		`CREATE INDEX IF NOT EXISTS idx_auth_codes_expires ON auth_codes(expires_at)`,
+		// Migration: Add source_type and source_path columns to job_templates table.
+		`ALTER TABLE job_templates ADD COLUMN source_type TEXT NOT NULL DEFAULT 'inline'`,
+		`ALTER TABLE job_templates ADD COLUMN source_path TEXT NOT NULL DEFAULT ''`,
 	}
 
 	for _, migration := range migrations {
@@ -467,10 +470,11 @@ func (s *SQLiteStore) CreateJobTemplate(ctx context.Context, template *JobTempla
 	}
 
 	_, err = s.db.ExecContext(ctx, `
-		INSERT INTO job_templates (id, group_id, name, owner, repo, workflow_id, ref, default_inputs, labels, in_config, created_at, updated_at)
-		VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+		INSERT INTO job_templates (id, group_id, name, owner, repo, workflow_id, ref, default_inputs, labels, in_config, source_type, source_path, created_at, updated_at)
+		VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
 	`, template.ID, template.GroupID, template.Name, template.Owner, template.Repo,
-		template.WorkflowID, template.Ref, string(inputsJSON), string(labelsJSON), template.InConfig, template.CreatedAt, template.UpdatedAt)
+		template.WorkflowID, template.Ref, string(inputsJSON), string(labelsJSON), template.InConfig,
+		template.SourceType, template.SourcePath, template.CreatedAt, template.UpdatedAt)
 
 	if err != nil {
 		return fmt.Errorf("inserting job_template: %w", err)
@@ -488,11 +492,11 @@ func (s *SQLiteStore) GetJobTemplate(ctx context.Context, id string) (*JobTempla
 	var inConfig int
 
 	err := s.db.QueryRowContext(ctx, `
-		SELECT id, group_id, name, owner, repo, workflow_id, ref, default_inputs, labels, in_config, created_at, updated_at
+		SELECT id, group_id, name, owner, repo, workflow_id, ref, default_inputs, labels, in_config, source_type, source_path, created_at, updated_at
 		FROM job_templates WHERE id = ?
 	`, id).Scan(&template.ID, &template.GroupID, &template.Name, &template.Owner,
 		&template.Repo, &template.WorkflowID, &template.Ref, &inputsJSON, &labelsJSON,
-		&inConfig, &template.CreatedAt, &template.UpdatedAt)
+		&inConfig, &template.SourceType, &template.SourcePath, &template.CreatedAt, &template.UpdatedAt)
 
 	if err == sql.ErrNoRows {
 		return nil, nil
@@ -522,7 +526,7 @@ func (s *SQLiteStore) GetJobTemplate(ctx context.Context, id string) (*JobTempla
 // ListJobTemplatesByGroup retrieves all job templates for a group.
 func (s *SQLiteStore) ListJobTemplatesByGroup(ctx context.Context, groupID string) ([]*JobTemplate, error) {
 	rows, err := s.db.QueryContext(ctx, `
-		SELECT id, group_id, name, owner, repo, workflow_id, ref, default_inputs, labels, in_config, created_at, updated_at
+		SELECT id, group_id, name, owner, repo, workflow_id, ref, default_inputs, labels, in_config, source_type, source_path, created_at, updated_at
 		FROM job_templates WHERE group_id = ? ORDER BY name
 	`, groupID)
 	if err != nil {
@@ -542,7 +546,7 @@ func (s *SQLiteStore) ListJobTemplatesByGroup(ctx context.Context, groupID strin
 
 		if err := rows.Scan(&template.ID, &template.GroupID, &template.Name, &template.Owner,
 			&template.Repo, &template.WorkflowID, &template.Ref, &inputsJSON, &labelsJSON,
-			&inConfig, &template.CreatedAt, &template.UpdatedAt); err != nil {
+			&inConfig, &template.SourceType, &template.SourcePath, &template.CreatedAt, &template.UpdatedAt); err != nil {
 			return nil, fmt.Errorf("scanning job_template: %w", err)
 		}
 
@@ -580,10 +584,10 @@ func (s *SQLiteStore) UpdateJobTemplate(ctx context.Context, template *JobTempla
 	template.UpdatedAt = time.Now()
 
 	_, err = s.db.ExecContext(ctx, `
-		UPDATE job_templates SET name = ?, owner = ?, repo = ?, workflow_id = ?, ref = ?, default_inputs = ?, labels = ?, in_config = ?, updated_at = ?
+		UPDATE job_templates SET name = ?, owner = ?, repo = ?, workflow_id = ?, ref = ?, default_inputs = ?, labels = ?, in_config = ?, source_type = ?, source_path = ?, updated_at = ?
 		WHERE id = ?
 	`, template.Name, template.Owner, template.Repo, template.WorkflowID, template.Ref,
-		string(inputsJSON), string(labelsJSON), template.InConfig, template.UpdatedAt, template.ID)
+		string(inputsJSON), string(labelsJSON), template.InConfig, template.SourceType, template.SourcePath, template.UpdatedAt, template.ID)
 
 	if err != nil {
 		return fmt.Errorf("updating job_template: %w", err)

pkg/store/store.go

@@ -115,6 +115,8 @@ type JobTemplate struct {
 	DefaultInputs map[string]string `json:"default_inputs"`
 	Labels        map[string]string `json:"labels"`
 	InConfig      bool              `json:"in_config"`
+	SourceType    string            `json:"source_type"` // "inline", "file", or "url"
+	SourcePath    string            `json:"source_path"` // filename or URL (empty for inline)
 	CreatedAt     time.Time         `json:"created_at"`
 	UpdatedAt     time.Time         `json:"updated_at"`
 }

ui/src/pages/ApiDocsPage.tsx

@@ -22,6 +22,7 @@ export function ApiDocsPage() {
         setSidebarCollapsed(false);
       }
     };
+    // eslint-disable-next-line react-hooks/exhaustive-deps -- intentionally run only on mount/unmount
   }, []);
 
   const config: ReferenceProps['configuration'] = {

ui/src/pages/GroupPage.tsx

@@ -1554,6 +1554,30 @@ export function GroupPage() {
                                 Not in config
                               </span>
                             )}
+                            {template.source_type === 'file' && (
+                              <span
+                                className="inline-flex items-center gap-1 rounded-sm bg-blue-500/20 px-1.5 py-0.5 text-xs text-blue-300"
+                                title={`Loaded from local file: ${template.source_path}`}
+                              >
+                                <svg className="size-3" fill="none" stroke="currentColor" viewBox="0 0 24 24">
+                                  <path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2}
+                                    d="M9 12h6m-6 4h6m2 5H7a2 2 0 01-2-2V5a2 2 0 012-2h5.586a1 1 0 01.707.293l5.414 5.414a1 1 0 01.293.707V19a2 2 0 01-2 2z" />
+                                </svg>
+                                {template.source_path.split('/').pop()}
+                              </span>
+                            )}
+                            {template.source_type === 'url' && (
+                              <span
+                                className="inline-flex items-center gap-1 rounded-sm bg-purple-500/20 px-1.5 py-0.5 text-xs text-purple-300"
+                                title={`Loaded from URL: ${template.source_path}`}
+                              >
+                                <svg className="size-3" fill="none" stroke="currentColor" viewBox="0 0 24 24">
+                                  <path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2}
+                                    d="M13.828 10.172a4 4 0 00-5.656 0l-4 4a4 4 0 105.656 5.656l1.102-1.101m-.758-4.899a4 4 0 005.656 0l4-4a4 4 0 00-5.656-5.656l-1.1 1.1" />
+                                </svg>
+                                URL
+                              </span>
+                            )}
                           </div>
                           {/* Template labels */}
                           {template.labels && Object.keys(template.labels).length > 0 && (