Skip to content

Commit f0fef55

Browse files
artaasadiartaasadi
committed
fix: fix named queries on migrator without compliance
1 parent f3af3d3 commit f0fef55

File tree

3 files changed

+81
-128
lines changed

3 files changed

+81
-128
lines changed

‎jobs/post-install-job/job/migrations/compliance/git_parser.go‎

Lines changed: 0 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,6 @@ type GitParser struct {
3030
controls []db.Control
3131
policies []db.Policy
3232
policyParamValues []models.PolicyParameterValues
33-
queryViews []models.QueryView
34-
coreServiceQueries []models.Query
3533
controlsPolicies map[string]db.Policy
3634
namedPolicies map[string]NamedQuery
3735
Comparison *git.ComparisonResultGrouped
@@ -65,37 +63,6 @@ func populateMdMapFromPath(path string) (map[string]string, error) {
6563
return result, nil
6664
}
6765

68-
func (g *GitParser) ExtractNamedQueries() error {
69-
err := filepath.Walk(config.QueriesGitPath, func(path string, info fs.FileInfo, err error) error {
70-
if !info.IsDir() && strings.HasSuffix(path, ".yaml") {
71-
id := strings.TrimSuffix(info.Name(), ".yaml")
72-
73-
content, err := os.ReadFile(path)
74-
if err != nil {
75-
return err
76-
}
77-
78-
var item NamedQuery
79-
err = yaml.Unmarshal(content, &item)
80-
if err != nil {
81-
g.logger.Error("failure in unmarshal", zap.String("path", path), zap.Error(err))
82-
return nil
83-
}
84-
85-
if item.ID != "" {
86-
id = item.ID
87-
}
88-
89-
g.namedPolicies[id] = item
90-
}
91-
return nil
92-
})
93-
if err != nil {
94-
return err
95-
}
96-
return nil
97-
}
98-
9966
func (g *GitParser) ExtractControls(complianceControlsPath string, controlEnrichmentBasePath string) error {
10067
var err error
10168

@@ -791,9 +758,6 @@ func (g *GitParser) ExtractCompliance(compliancePath string, controlEnrichmentBa
791758
return err
792759
}
793760

794-
if err := g.ExtractNamedQueries(); err != nil {
795-
return err
796-
}
797761
if err := g.ExtractPolicies(path.Join(compliancePath, "policies")); err != nil {
798762
return err
799763
}

‎jobs/post-install-job/job/migrations/compliance/populate.go‎

Lines changed: 0 additions & 90 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ import (
99
"github.com/opengovern/og-util/pkg/httpclient"
1010
"github.com/opengovern/og-util/pkg/model"
1111
"github.com/opengovern/opensecurity/jobs/post-install-job/utils"
12-
coreClient "github.com/opengovern/opensecurity/services/core/client"
1312
"github.com/opengovern/opensecurity/services/core/db/models"
1413
integrationClient "github.com/opengovern/opensecurity/services/integration/client"
1514
"io/fs"
@@ -26,8 +25,6 @@ import (
2625
"gorm.io/gorm/clause"
2726
)
2827

29-
var QueryParameters []models.PolicyParameterValues
30-
3128
type Migration struct {
3229
}
3330

@@ -227,98 +224,11 @@ func (m Migration) Run(ctx context.Context, conf config.MigratorConfig, logger *
227224
return err
228225
}
229226

230-
loadedQueryViewsQueries := make(map[string]bool)
231-
missingQueryViewsQueries := make(map[string]bool)
232-
err = dbCore.Orm.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
233-
for _, obj := range p.coreServiceQueries {
234-
obj.QueryViews = nil
235-
err := tx.Clauses(clause.OnConflict{
236-
Columns: []clause.Column{{Name: "id"}}, // key column
237-
DoNothing: true,
238-
}).Create(&obj).Error
239-
if err != nil {
240-
return err
241-
}
242-
for _, param := range obj.Parameters {
243-
err = tx.Clauses(clause.OnConflict{
244-
Columns: []clause.Column{{Name: "key"}, {Name: "query_id"}}, // key columns
245-
DoNothing: true,
246-
}).Create(&param).Error
247-
if err != nil {
248-
return fmt.Errorf("failure in query parameter insert: %v", err)
249-
}
250-
}
251-
loadedQueryViewsQueries[obj.ID] = true
252-
}
253-
254-
return nil
255-
})
256-
if err != nil {
257-
logger.Error("failed to insert query views", zap.Error(err))
258-
return err
259-
}
260-
261-
err = dbCore.Orm.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
262-
for _, obj := range p.queryViews {
263-
if obj.QueryID != nil && !loadedQueryViewsQueries[*obj.QueryID] {
264-
missingQueryViewsQueries[*obj.QueryID] = true
265-
logger.Info("query not found", zap.String("query_id", *obj.QueryID))
266-
continue
267-
}
268-
err := tx.Clauses(clause.OnConflict{
269-
Columns: []clause.Column{{Name: "id"}},
270-
DoNothing: true,
271-
}).Create(&obj).Error
272-
if err != nil {
273-
logger.Error("error while inserting query view", zap.Error(err))
274-
return err
275-
}
276-
for _, tag := range obj.Tags {
277-
err = tx.Clauses(clause.OnConflict{
278-
Columns: []clause.Column{{Name: "key"}, {Name: "query_view_id"}}, // key columns
279-
DoNothing: true,
280-
}).Create(&tag).Error
281-
if err != nil {
282-
return fmt.Errorf("failure in control tag insert: %v", err)
283-
}
284-
}
285-
}
286-
287-
return nil
288-
})
289-
if err != nil {
290-
logger.Error("failed to insert query views", zap.Error(err))
291-
return err
292-
}
293-
294227
err = populateQueries(ctx, logger, dbCore, conf)
295228
if err != nil {
296229
return err
297230
}
298231

299-
err = dbCore.Orm.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
300-
for _, obj := range QueryParameters {
301-
err := tx.Clauses(clause.OnConflict{
302-
DoNothing: true,
303-
}).Create(&obj).Error
304-
if err != nil {
305-
return err
306-
}
307-
}
308-
return nil
309-
})
310-
if err != nil {
311-
logger.Error("failed to insert query params", zap.Error(err))
312-
return err
313-
}
314-
315-
mClient := coreClient.NewCoreServiceClient(conf.Core.BaseURL)
316-
err = mClient.ReloadViews(&httpclient.Context{Ctx: ctx, UserRole: authApi.AdminRole})
317-
if err != nil {
318-
logger.Error("failed to reload views", zap.Error(err))
319-
return fmt.Errorf("failed to reload views: %s", err.Error())
320-
}
321-
322232
return nil
323233
}
324234

‎jobs/post-install-job/job/migrations/inventory/migrator.go‎

Lines changed: 81 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,12 @@ import (
55
"encoding/json"
66
"fmt"
77
"github.com/goccy/go-yaml"
8+
authApi "github.com/opengovern/og-util/pkg/api"
9+
"github.com/opengovern/og-util/pkg/httpclient"
810
"github.com/opengovern/og-util/pkg/integration"
911
"github.com/opengovern/opensecurity/jobs/post-install-job/utils"
1012
"github.com/opengovern/opensecurity/pkg/types"
13+
coreClient "github.com/opengovern/opensecurity/services/core/client"
1114
"io/fs"
1215
"os"
1316
"path"
@@ -164,12 +167,19 @@ func (m Migration) Run(ctx context.Context, conf config.MigratorConfig, logger *
164167
return fmt.Errorf("failure in azure transaction: %v", err)
165168
}
166169

167-
err = ExtractQueryViews(ctx, logger, dbm, config.QueryViewsGitPath)
170+
err = ExtractQueryViews(ctx, logger, dbm, conf, config.QueryViewsGitPath)
171+
if err != nil {
172+
return err
173+
}
174+
err = ExtractNamedQueries(ctx, logger, dbm)
175+
if err != nil {
176+
return err
177+
}
168178

169179
return nil
170180
}
171181

172-
func ExtractQueryViews(ctx context.Context, logger *zap.Logger, dbm db.Database, viewsPath string) error {
182+
func ExtractQueryViews(ctx context.Context, logger *zap.Logger, dbm db.Database, conf config.MigratorConfig, viewsPath string) error {
173183
var queries []models.Query
174184
var queryViews []models.QueryView
175185
err := filepath.WalkDir(viewsPath, func(path string, d fs.DirEntry, err error) error {
@@ -258,5 +268,74 @@ func ExtractQueryViews(ctx context.Context, logger *zap.Logger, dbm db.Database,
258268
return nil
259269
})
260270

271+
mClient := coreClient.NewCoreServiceClient(conf.Core.BaseURL)
272+
err = mClient.ReloadViews(&httpclient.Context{Ctx: ctx, UserRole: authApi.AdminRole})
273+
if err != nil {
274+
logger.Error("failed to reload views", zap.Error(err))
275+
return fmt.Errorf("failed to reload views: %s", err.Error())
276+
}
277+
261278
return err
262279
}
280+
281+
func ExtractNamedQueries(ctx context.Context, logger *zap.Logger, dbm db.Database) error {
282+
var queries []NamedQuery
283+
err := filepath.Walk(config.QueriesGitPath, func(path string, info fs.FileInfo, err error) error {
284+
if !info.IsDir() && strings.HasSuffix(path, ".yaml") {
285+
id := strings.TrimSuffix(info.Name(), ".yaml")
286+
287+
content, err := os.ReadFile(path)
288+
if err != nil {
289+
return err
290+
}
291+
292+
var item NamedQuery
293+
err = yaml.Unmarshal(content, &item)
294+
if err != nil {
295+
logger.Error("failure in unmarshal", zap.String("path", path), zap.Error(err))
296+
return nil
297+
}
298+
299+
if item.ID == "" {
300+
item.ID = id
301+
}
302+
303+
queries = append(queries, item)
304+
}
305+
return nil
306+
})
307+
if err != nil {
308+
return err
309+
}
310+
311+
err = dbm.ORM.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
312+
for _, q := range queries {
313+
err = tx.Clauses(clause.OnConflict{
314+
Columns: []clause.Column{{Name: "id"}}, // key column
315+
DoNothing: true,
316+
}).Create(&q).Error
317+
if err != nil {
318+
return err
319+
}
320+
for k, v := range q.Tags {
321+
tag := models.NamedQueryTag{
322+
NamedQueryID: q.ID,
323+
Tag: model.Tag{
324+
Key: k,
325+
Value: v,
326+
},
327+
}
328+
err = tx.Clauses(clause.OnConflict{
329+
Columns: []clause.Column{{Name: "named_query_id"}, {Name: "key"}}, // key column
330+
DoNothing: true,
331+
}).Create(&tag).Error
332+
if err != nil {
333+
return err
334+
}
335+
}
336+
}
337+
return nil
338+
})
339+
340+
return nil
341+
}

0 commit comments

Comments
 (0)