Skip to content

Commit 2da7049

Browse files
zemberdotnetzemberdotnet
authored
Merge pull request #951 from JupiterOne/halo-161-logging-improvements
Logging Improvement / Error Updates
2 parents 472ca01 + c36beae commit 2da7049

File tree

23 files changed

+122
-211
lines changed

23 files changed

+122
-211
lines changed

‎CHANGELOG.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,12 @@ and this project adheres to
99

1010
## Unreleased
1111

12+
## 10.5.3 - 2023-09-18
13+
14+
- Use the same logger for all logs created via the `j1-integration run` command
15+
to improve formatting consistency
16+
- Update internal `err` log serializer
17+
1218
## 10.5.2 - 2023-09-14
1319

1420
- Improvements to integration graph generator

‎lerna.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,5 +4,5 @@
44
"packages/integration-sdk-*",
55
"packages/cli"
66
],
7-
"version": "10.5.2"
7+
"version": "10.5.3"
88
}

‎packages/cli/package.json‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@jupiterone/cli",
3-
"version": "10.5.2",
3+
"version": "10.5.3",
44
"description": "The JupiterOne cli",
55
"main": "dist/src/index.js",
66
"types": "dist/src/index.d.ts",
@@ -24,8 +24,8 @@
2424
"test": "jest"
2525
},
2626
"dependencies": {
27-
"@jupiterone/integration-sdk-core": "^10.5.2",
28-
"@jupiterone/integration-sdk-runtime": "^10.5.2",
27+
"@jupiterone/integration-sdk-core": "^10.5.3",
28+
"@jupiterone/integration-sdk-runtime": "^10.5.3",
2929
"@lifeomic/attempt": "^3.0.3",
3030
"commander": "^5.0.0",
3131
"globby": "^11.0.1",

‎packages/integration-sdk-benchmark/package.json‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@jupiterone/integration-sdk-benchmark",
3-
"version": "10.5.2",
3+
"version": "10.5.3",
44
"private": true,
55
"description": "SDK benchmarking scripts",
66
"main": "./src/index.js",
@@ -15,8 +15,8 @@
1515
"benchmark": "for file in ./src/benchmarks/*; do yarn prebenchmark && node $file; done"
1616
},
1717
"dependencies": {
18-
"@jupiterone/integration-sdk-core": "^10.5.2",
19-
"@jupiterone/integration-sdk-runtime": "^10.5.2",
18+
"@jupiterone/integration-sdk-core": "^10.5.3",
19+
"@jupiterone/integration-sdk-runtime": "^10.5.3",
2020
"benchmark": "^2.1.4"
2121
}
2222
}

‎packages/integration-sdk-cli/package.json‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@jupiterone/integration-sdk-cli",
3-
"version": "10.5.2",
3+
"version": "10.5.3",
44
"description": "The SDK for developing JupiterOne integrations",
55
"main": "dist/src/index.js",
66
"types": "dist/src/index.d.ts",
@@ -25,8 +25,8 @@
2525
},
2626
"dependencies": {
2727
"@jupiterone/data-model": "^0.54.0",
28-
"@jupiterone/integration-sdk-core": "^10.5.2",
29-
"@jupiterone/integration-sdk-runtime": "^10.5.2",
28+
"@jupiterone/integration-sdk-core": "^10.5.3",
29+
"@jupiterone/integration-sdk-runtime": "^10.5.3",
3030
"chalk": "^4",
3131
"commander": "^9.4.0",
3232
"fs-extra": "^10.1.0",
@@ -44,7 +44,7 @@
4444
"vis": "^4.21.0-EOL"
4545
},
4646
"devDependencies": {
47-
"@jupiterone/integration-sdk-private-test-utils": "^10.5.2",
47+
"@jupiterone/integration-sdk-private-test-utils": "^10.5.3",
4848
"@pollyjs/adapter-node-http": "^6.0.5",
4949
"@pollyjs/core": "^6.0.5",
5050
"@pollyjs/persister-fs": "^6.0.5",

‎packages/integration-sdk-cli/src/__tests__/cli-run-failure.test.ts‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,9 @@ import { Polly } from '@pollyjs/core';
22
import NodeHttpAdapter from '@pollyjs/adapter-node-http';
33
import FSPersister from '@pollyjs/persister-fs';
44
import { loadProjectStructure } from '@jupiterone/integration-sdk-private-test-utils';
5-
import { SynchronizationJobStatus } from '@jupiterone/integration-sdk-core';
65
import { generateSynchronizationJob } from './util/synchronization';
76
import { createCli } from '../index';
87
import { setupSynchronizerApi } from './util/synchronization';
9-
import * as log from '../log';
108
import { createTestPolly } from './util/recording';
119

1210
jest.mock('../log');
@@ -37,17 +35,20 @@ test('aborts synchronization job if an error occurs', async () => {
3735

3836
setupSynchronizerApi({ polly, job, baseUrl: 'https://api.us.jupiterone.io' });
3937

38+
let calledAbort = false;
39+
polly.server
40+
.post(
41+
`https://api.us.jupiterone.io/persister/synchronization/jobs/${job.id}/abort`,
42+
)
43+
.intercept((req, res) => {
44+
calledAbort = true;
45+
});
4046
await createCli().parseAsync([
4147
'node',
4248
'j1-integration',
4349
'run',
4450
'--integrationInstanceId',
4551
'test',
4652
]);
47-
48-
expect(log.displaySynchronizationResults).toHaveBeenCalledTimes(1);
49-
expect(log.displaySynchronizationResults).toHaveBeenCalledWith({
50-
...job,
51-
status: SynchronizationJobStatus.ABORTED,
52-
});
53+
expect(calledAbort).toBe(true);
5354
});

‎packages/integration-sdk-cli/src/__tests__/cli-run.test.ts‎

Lines changed: 22 additions & 140 deletions
Original file line numberDiff line numberDiff line change
@@ -10,12 +10,6 @@ import {
1010
setupSynchronizerApi,
1111
} from './util/synchronization';
1212

13-
import {
14-
StepResultStatus,
15-
SynchronizationJobStatus,
16-
} from '@jupiterone/integration-sdk-core';
17-
18-
import * as log from '../log';
1913
import { createTestPolly } from './util/recording';
2014

2115
jest.mock('../log');
@@ -80,92 +74,19 @@ test('disables graph object schema validation', async () => {
8074
expect(process.env.ENABLE_GRAPH_OBJECT_SCHEMA_VALIDATION).toBeUndefined();
8175
});
8276

83-
test('step should fail if enableSchemaValidation = true', async () => {
84-
loadProjectStructure('instanceWithNonValidatingSteps');
85-
const job = generateSynchronizationJob();
86-
87-
setupSynchronizerApi({
88-
polly,
89-
job,
90-
baseUrl: 'https://api.us.jupiterone.io',
91-
});
92-
93-
await createCli().parseAsync([
94-
'node',
95-
'j1-integration',
96-
'run',
97-
'--integrationInstanceId',
98-
'test',
99-
]);
100-
101-
expect(log.displaySynchronizationResults).toHaveBeenCalledTimes(1);
102-
103-
expect(log.displayExecutionResults).toHaveBeenCalledTimes(1);
104-
expect(log.displayExecutionResults).toHaveBeenCalledWith({
105-
integrationStepResults: [
106-
{
107-
id: 'fetch-users',
108-
name: 'Fetch Users',
109-
declaredTypes: ['my_user'],
110-
partialTypes: [],
111-
encounteredTypes: [],
112-
status: StepResultStatus.FAILURE,
113-
},
114-
],
115-
metadata: {
116-
partialDatasets: {
117-
types: ['my_user'],
118-
},
119-
},
120-
});
121-
});
122-
123-
test('step should pass if enableSchemaValidation = false', async () => {
124-
loadProjectStructure('instanceWithNonValidatingSteps');
125-
const job = generateSynchronizationJob();
126-
127-
setupSynchronizerApi({
128-
polly,
129-
job,
130-
baseUrl: 'https://api.us.jupiterone.io',
131-
});
132-
133-
await createCli().parseAsync([
134-
'node',
135-
'j1-integration',
136-
'run',
137-
'--integrationInstanceId',
138-
'test',
139-
'--disable-schema-validation',
140-
]);
141-
142-
expect(log.displaySynchronizationResults).toHaveBeenCalledTimes(1);
143-
144-
expect(log.displayExecutionResults).toHaveBeenCalledTimes(1);
145-
expect(log.displayExecutionResults).toHaveBeenCalledWith({
146-
integrationStepResults: [
147-
{
148-
id: 'fetch-users',
149-
name: 'Fetch Users',
150-
declaredTypes: ['my_user'],
151-
partialTypes: [],
152-
encounteredTypes: ['my_user'],
153-
status: StepResultStatus.SUCCESS,
154-
},
155-
],
156-
metadata: {
157-
partialDatasets: {
158-
types: [],
159-
},
160-
},
161-
});
162-
});
163-
16477
test('executes integration and performs upload', async () => {
16578
const job = generateSynchronizationJob();
16679

16780
setupSynchronizerApi({ polly, job, baseUrl: 'https://api.us.jupiterone.io' });
16881

82+
let calledFinalize = false;
83+
polly.server
84+
.post(
85+
`https://api.us.jupiterone.io/persister/synchronization/jobs/${job.id}/finalize`,
86+
)
87+
.intercept((req, res) => {
88+
calledFinalize = true;
89+
});
16990
await createCli().parseAsync([
17091
'node',
17192
'j1-integration',
@@ -174,43 +95,7 @@ test('executes integration and performs upload', async () => {
17495
'test',
17596
]);
17697

177-
expect(log.displayExecutionResults).toHaveBeenCalledTimes(1);
178-
expect(log.displayExecutionResults).toHaveBeenCalledWith({
179-
integrationStepResults: [
180-
{
181-
id: 'fetch-accounts',
182-
name: 'Fetch Accounts',
183-
declaredTypes: ['my_account'],
184-
partialTypes: [],
185-
encounteredTypes: ['my_account'],
186-
status: StepResultStatus.SUCCESS,
187-
},
188-
{
189-
id: 'fetch-users',
190-
name: 'Fetch Users',
191-
declaredTypes: ['my_user', 'my_account_has_user'],
192-
partialTypes: [],
193-
encounteredTypes: ['my_user', 'my_account_has_user'],
194-
status: StepResultStatus.SUCCESS,
195-
},
196-
],
197-
metadata: {
198-
partialDatasets: {
199-
types: [],
200-
},
201-
},
202-
});
203-
204-
expect(log.displaySynchronizationResults).toHaveBeenCalledTimes(1);
205-
expect(log.displaySynchronizationResults).toHaveBeenCalledWith({
206-
...job,
207-
status: SynchronizationJobStatus.FINALIZE_PENDING,
208-
// These are the expected number of entities and relationships
209-
// collected when executing the
210-
// 'typeScriptIntegrationProject' fixture
211-
numEntitiesUploaded: 2,
212-
numRelationshipsUploaded: 1,
213-
});
98+
expect(calledFinalize).toBe(true);
21499
});
215100

216101
test('executes integration and skips finalization with skip-finalize', async () => {
@@ -222,6 +107,15 @@ test('executes integration and skips finalization with skip-finalize', async ()
222107
baseUrl: 'https://api.us.jupiterone.io',
223108
});
224109

110+
let calledFinalize = false;
111+
polly.server
112+
.post(
113+
`https://api.us.jupiterone.io/persister/synchronization/jobs/${job.id}/finalize`,
114+
)
115+
.intercept((req, res) => {
116+
calledFinalize = true;
117+
});
118+
225119
await createCli().parseAsync([
226120
'node',
227121
'j1-integration',
@@ -230,19 +124,7 @@ test('executes integration and skips finalization with skip-finalize', async ()
230124
'test',
231125
'--skip-finalize',
232126
]);
233-
234-
expect(log.displayExecutionResults).toHaveBeenCalledTimes(1);
235-
236-
expect(log.displaySynchronizationResults).toHaveBeenCalledTimes(1);
237-
expect(log.displaySynchronizationResults).toHaveBeenCalledWith({
238-
...job,
239-
status: SynchronizationJobStatus.AWAITING_UPLOADS,
240-
// These are the expected number of entities and relationships
241-
// collected when executing the
242-
// 'typeScriptIntegrationProject' fixture
243-
numEntitiesUploaded: 2,
244-
numRelationshipsUploaded: 1,
245-
});
127+
expect(calledFinalize).toBe(false);
246128
});
247129

248130
test('does not publish events for source "api" since there is no integrationJobId', async () => {
@@ -256,7 +138,9 @@ test('does not publish events for source "api" since there is no integrationJobI
256138

257139
let eventsPublished = false;
258140
polly.server
259-
.post(`https://example.com/persister/synchronization/jobs/${job.id}/events`)
141+
.post(
142+
`https://api.us.jupiterone.io/persister/synchronization/jobs/${job.id}/events`,
143+
)
260144
.intercept((req, res) => {
261145
eventsPublished = true;
262146
});
@@ -272,8 +156,6 @@ test('does not publish events for source "api" since there is no integrationJobI
272156
]);
273157

274158
expect(eventsPublished).toBe(false);
275-
expect(log.displayExecutionResults).toHaveBeenCalledTimes(1);
276-
expect(log.displaySynchronizationResults).toHaveBeenCalledTimes(1);
277159
});
278160

279161
test('should use JUPITERONE_API_KEY value in Authorization request header', async () => {

‎packages/integration-sdk-cli/src/__tests__/util/synchronization.ts‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -97,8 +97,14 @@ export function generateSynchronizationJob(
9797
id: 'test',
9898
source: options?.source || 'integration-managed',
9999
scope: options?.scope,
100-
integrationJobId: options?.integrationJobId || 'test-job-id',
101-
integrationInstanceId: options?.integrationInstanceId || 'test-instance-id',
100+
integrationJobId:
101+
options?.source === 'api'
102+
? undefined
103+
: options?.integrationJobId || 'test-job-id',
104+
integrationInstanceId:
105+
options?.source === 'api'
106+
? undefined
107+
: options?.integrationInstanceId || 'test-instance-id',
102108
status: SynchronizationJobStatus.AWAITING_UPLOADS,
103109
startTimestamp: Date.now(),
104110
numEntitiesUploaded: 0,

0 commit comments

Comments
 (0)