fix: fix named queries on migrator without compliance

Author: artaasadi

jobs/post-install-job/job/migrations/compliance/populate.go

@@ -2,20 +2,7 @@ package compliance
 
 import (
 	"context"
-	"errors"
 	"fmt"
-	"github.com/goccy/go-yaml"
-	authApi "github.com/opengovern/og-util/pkg/api"
-	"github.com/opengovern/og-util/pkg/httpclient"
-	"github.com/opengovern/og-util/pkg/model"
-	"github.com/opengovern/opensecurity/jobs/post-install-job/utils"
-	"github.com/opengovern/opensecurity/services/core/db/models"
-	integrationClient "github.com/opengovern/opensecurity/services/integration/client"
-	"io/fs"
-	"os"
-	"path/filepath"
-	"strings"
-
 	"github.com/opengovern/og-util/pkg/postgres"
 	"github.com/opengovern/opensecurity/jobs/post-install-job/config"
 	"github.com/opengovern/opensecurity/services/compliance/db"
@@ -223,185 +210,5 @@ func (m Migration) Run(ctx context.Context, conf config.MigratorConfig, logger *
 		logger.Info("inserted controls and benchmarks", zap.Error(err))
 		return err
 	}
-
-	err = populateQueries(ctx, logger, dbCore, conf)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func populateQueries(ctx context.Context, logger *zap.Logger, db db.Database, conf config.MigratorConfig) error {
-	iClient := integrationClient.NewIntegrationServiceClient(conf.Integration.BaseURL)
-	pluginTables, err := iClient.GetPluginsTables(&httpclient.Context{Ctx: ctx, UserRole: authApi.AdminRole})
-	if err != nil {
-		logger.Error("failed to get plugin tables", zap.Error(err))
-		return nil
-	}
-	tablesPluginMap := make(map[string]string)
-	for _, p := range pluginTables {
-		for _, t := range p.Tables {
-			tablesPluginMap[t] = p.PluginID
-		}
-	}
-
-	err = db.Orm.Transaction(func(tx *gorm.DB) error {
-		err := filepath.Walk(config.QueriesGitPath, func(path string, info fs.FileInfo, err error) error {
-			if !info.IsDir() && strings.HasSuffix(path, ".yaml") {
-				return populateFinderItem(logger, tx, path, info, tablesPluginMap)
-			}
-			return nil
-		})
-		if err != nil && !errors.Is(err, fs.ErrNotExist) {
-			logger.Error("failed to get queries", zap.Error(err))
-			return err
-		}
-		return nil
-	})
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func populateFinderItem(logger *zap.Logger, tx *gorm.DB, path string, info fs.FileInfo, tablesPluginMap map[string]string) error {
-	id := strings.TrimSuffix(info.Name(), ".yaml")
-
-	content, err := os.ReadFile(path)
-	if err != nil {
-		return err
-	}
-
-	var item NamedQuery
-	err = yaml.Unmarshal(content, &item)
-	if err != nil {
-		logger.Error("failure in unmarshal", zap.String("path", path), zap.Error(err))
-		return err
-	}
-
-	if item.ID != "" {
-		id = item.ID
-	}
-
-	var integrationTypes []string
-	for _, c := range item.IntegrationTypes {
-		integrationTypes = append(integrationTypes, string(c))
-	}
-
-	isBookmarked := false
-	tags := make([]models.NamedQueryTag, 0, len(item.Tags))
-	for k, v := range item.Tags {
-		if k == "platform_queries_bookmark" {
-			isBookmarked = true
-		}
-		tag := models.NamedQueryTag{
-			NamedQueryID: id,
-			Tag: model.Tag{
-				Key:   k,
-				Value: v,
-			},
-		}
-		tags = append(tags, tag)
-	}
-
-	listOfTables, err := utils.ExtractTableRefsFromPolicy("sql", item.Query)
-	if err != nil {
-		logger.Error("failed to extract table refs from query", zap.String("query-id", id), zap.Error(err))
-	}
-	if len(integrationTypes) == 0 {
-		integrationTypesMap := make(map[string]bool)
-		for _, t := range listOfTables {
-			if v, ok := tablesPluginMap[t]; ok {
-				integrationTypesMap[v] = true
-			}
-		}
-		for it := range integrationTypesMap {
-			integrationTypes = append(integrationTypes, it)
-		}
-	}
-
-	namedQuery := models.NamedQuery{
-		ID:               id,
-		IntegrationTypes: integrationTypes,
-		Title:            item.Title,
-		Description:      item.Description,
-		IsBookmarked:     isBookmarked,
-		QueryID:          &id,
-	}
-
-	parameters, err := utils.ExtractParameters("sql", item.Query)
-	if err != nil {
-		logger.Error("extract control failed: failed to extract parameters from query", zap.String("control-id", namedQuery.ID), zap.Error(err))
-		return nil
-	}
-	queryParams := []models.QueryParameter{}
-	for _, p := range parameters {
-		queryParams = append(queryParams, models.QueryParameter{
-			QueryID: namedQuery.ID,
-			Key:     p,
-		})
-	}
-
-	query := models.Query{
-		ID:             namedQuery.ID,
-		QueryToExecute: item.Query,
-		ListOfTables:   listOfTables,
-		Engine:         "sql",
-		Parameters:     queryParams,
-	}
-	err = tx.Clauses(clause.OnConflict{
-		Columns:   []clause.Column{{Name: "id"}}, // key column
-		DoNothing: true,
-	}).Create(&query).Error
-	if err != nil {
-		logger.Error("failure in Creating Policy", zap.String("query_id", id), zap.Error(err))
-		return err
-	}
-	for _, param := range query.Parameters {
-		err = tx.Clauses(clause.OnConflict{
-			Columns:   []clause.Column{{Name: "key"}, {Name: "query_id"}}, // key columns
-			DoNothing: true,
-		}).Create(&param).Error
-		if err != nil {
-			return fmt.Errorf("failure in query parameter insert: %v", err)
-		}
-	}
-
-	err = tx.Model(&models.NamedQuery{}).Clauses(clause.OnConflict{
-		Columns:   []clause.Column{{Name: "id"}}, // key column
-		DoNothing: true,                          // column needed to be updated
-	}).Create(namedQuery).Error
-	if err != nil {
-		logger.Error("failure in insert query", zap.Error(err))
-		return err
-	}
-
-	if len(tags) > 0 {
-		for _, tag := range tags {
-			err = tx.Model(&models.NamedQueryTag{}).Create(&tag).Error
-			if err != nil {
-				logger.Error("failure in insert tags", zap.Error(err))
-				return err
-			}
-		}
-	}
-
-	for _, p := range item.Parameters {
-		err := tx.Clauses(clause.OnConflict{
-			Columns: []clause.Column{{Name: "key"}, {Name: "control_id"}},
-			DoUpdates: clause.Assignments(map[string]interface{}{
-				"value": gorm.Expr("CASE WHEN policy_parameter_values.value = '' THEN ? ELSE policy_parameter_values.value END", p.Value),
-			}),
-		}).Create(&models.PolicyParameterValues{
-			Key:       p.Key,
-			ControlID: "",
-			Value:     p.Value,
-		}).Error
-		if err != nil {
-			return err
-		}
-	}
-
 	return nil
 }